2024-11-24 15:27:30,234 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 15:27:30,249 main DEBUG Took 0.012799 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-24 15:27:30,249 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-24 15:27:30,250 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-24 15:27:30,251 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-24 15:27:30,253 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,261 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-24 15:27:30,273 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,275 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,275 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,276 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,276 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,277 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,278 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,278 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,279 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,279 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,280 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,280 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,280 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,281 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,283 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,284 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,284 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-24 15:27:30,284 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,285 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-24 15:27:30,286 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-24 15:27:30,288 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-24 15:27:30,290 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-24 15:27:30,290 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-24 15:27:30,292 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-24 15:27:30,292 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-24 15:27:30,303 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-24 15:27:30,306 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-24 15:27:30,308 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-24 15:27:30,308 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-24 15:27:30,308 main DEBUG createAppenders(={Console}) 2024-11-24 15:27:30,309 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-11-24 15:27:30,309 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-11-24 15:27:30,310 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-11-24 15:27:30,310 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-24 15:27:30,311 main DEBUG OutputStream closed 2024-11-24 15:27:30,311 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-24 15:27:30,312 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-24 15:27:30,312 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-11-24 15:27:30,394 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-24 15:27:30,396 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-24 15:27:30,397 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-24 15:27:30,398 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-24 15:27:30,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-24 15:27:30,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-24 15:27:30,399 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-24 15:27:30,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-24 15:27:30,400 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-24 15:27:30,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-24 15:27:30,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-24 15:27:30,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-24 15:27:30,402 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-24 15:27:30,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-24 15:27:30,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-24 15:27:30,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-24 15:27:30,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-24 15:27:30,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-24 15:27:30,407 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24 15:27:30,407 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-11-24 15:27:30,408 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-24 15:27:30,408 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-11-24T15:27:30,679 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857 2024-11-24 15:27:30,682 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-24 15:27:30,682 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-24T15:27:30,692 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-24T15:27:30,697 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-11-24T15:27:30,720 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-24T15:27:30,773 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-24T15:27:30,774 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-24T15:27:30,791 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-24T15:27:30,813 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f, deleteOnExit=true 2024-11-24T15:27:30,814 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-24T15:27:30,815 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/test.cache.data in system properties and HBase conf 2024-11-24T15:27:30,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.tmp.dir in system properties and HBase conf 2024-11-24T15:27:30,816 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir in system properties and HBase conf 2024-11-24T15:27:30,817 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-24T15:27:30,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-24T15:27:30,818 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-24T15:27:30,928 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-24T15:27:31,041 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-24T15:27:31,045 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-24T15:27:31,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-24T15:27:31,046 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-24T15:27:31,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T15:27:31,047 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-24T15:27:31,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-24T15:27:31,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-24T15:27:31,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T15:27:31,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-24T15:27:31,049 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/nfs.dump.dir in system properties and HBase conf 2024-11-24T15:27:31,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/java.io.tmpdir in system properties and HBase conf 2024-11-24T15:27:31,050 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-24T15:27:31,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-24T15:27:31,051 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-24T15:27:31,950 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-24T15:27:32,023 INFO [Time-limited test {}] log.Log(170): Logging initialized @2621ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-24T15:27:32,100 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T15:27:32,167 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T15:27:32,216 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T15:27:32,216 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T15:27:32,218 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T15:27:32,237 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T15:27:32,240 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,AVAILABLE} 2024-11-24T15:27:32,242 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T15:27:32,434 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/java.io.tmpdir/jetty-localhost-39613-hadoop-hdfs-3_4_1-tests_jar-_-any-16383155520726349001/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T15:27:32,440 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:39613} 2024-11-24T15:27:32,440 INFO [Time-limited test {}] server.Server(415): Started @3039ms 2024-11-24T15:27:32,852 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T15:27:32,859 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T15:27:32,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T15:27:32,860 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T15:27:32,860 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T15:27:32,861 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,AVAILABLE} 2024-11-24T15:27:32,862 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T15:27:32,984 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/java.io.tmpdir/jetty-localhost-34689-hadoop-hdfs-3_4_1-tests_jar-_-any-8919536014309036362/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:27:32,985 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:34689} 2024-11-24T15:27:32,985 INFO [Time-limited test {}] server.Server(415): Started @3584ms 2024-11-24T15:27:33,040 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T15:27:33,178 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T15:27:33,185 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T15:27:33,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T15:27:33,189 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T15:27:33,189 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-24T15:27:33,191 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,AVAILABLE} 2024-11-24T15:27:33,192 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T15:27:33,339 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/java.io.tmpdir/jetty-localhost-45479-hadoop-hdfs-3_4_1-tests_jar-_-any-16451826462450841081/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:27:33,340 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:45479} 2024-11-24T15:27:33,341 INFO [Time-limited test {}] server.Server(415): Started @3939ms 2024-11-24T15:27:33,343 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T15:27:33,385 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-24T15:27:33,391 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-24T15:27:33,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-24T15:27:33,396 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-24T15:27:33,397 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-24T15:27:33,398 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,AVAILABLE} 2024-11-24T15:27:33,399 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-24T15:27:33,530 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data2/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,530 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data3/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,530 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data4/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,530 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data1/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,571 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/java.io.tmpdir/jetty-localhost-33835-hadoop-hdfs-3_4_1-tests_jar-_-any-1322931780021412476/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:27:33,572 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:33835} 2024-11-24T15:27:33,573 INFO [Time-limited test {}] server.Server(415): Started @4171ms 2024-11-24T15:27:33,576 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-24T15:27:33,579 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T15:27:33,579 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T15:27:33,651 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbeac9e7bffbbd184 with lease ID 0x4f0c6cd0eac05fa0: Processing first storage report for DS-0d56fc79-22a0-4846-addc-31a51dc0fa87 from datanode DatanodeRegistration(127.0.0.1:41867, datanodeUuid=f20b3dfe-433d-47b8-830a-34624f723627, infoPort=43739, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,652 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbeac9e7bffbbd184 with lease ID 0x4f0c6cd0eac05fa0: from storage DS-0d56fc79-22a0-4846-addc-31a51dc0fa87 node DatanodeRegistration(127.0.0.1:41867, datanodeUuid=f20b3dfe-433d-47b8-830a-34624f723627, infoPort=43739, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9705d764108db4e1 with lease ID 0x4f0c6cd0eac05f9f: Processing first storage report for DS-432632f3-225d-41fa-b045-26dfe1da38ff from datanode DatanodeRegistration(127.0.0.1:39259, datanodeUuid=3caae417-876c-4922-be6c-aa2097011217, infoPort=39897, infoSecurePort=0, ipcPort=38219, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9705d764108db4e1 with lease ID 0x4f0c6cd0eac05f9f: from storage DS-432632f3-225d-41fa-b045-26dfe1da38ff node DatanodeRegistration(127.0.0.1:39259, datanodeUuid=3caae417-876c-4922-be6c-aa2097011217, infoPort=39897, infoSecurePort=0, ipcPort=38219, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,653 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbeac9e7bffbbd184 with lease ID 0x4f0c6cd0eac05fa0: Processing first storage report for DS-b41298ba-06a9-4b8d-8a28-a7713fc0b66a from datanode DatanodeRegistration(127.0.0.1:41867, datanodeUuid=f20b3dfe-433d-47b8-830a-34624f723627, infoPort=43739, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbeac9e7bffbbd184 with lease ID 0x4f0c6cd0eac05fa0: from storage DS-b41298ba-06a9-4b8d-8a28-a7713fc0b66a node DatanodeRegistration(127.0.0.1:41867, datanodeUuid=f20b3dfe-433d-47b8-830a-34624f723627, infoPort=43739, infoSecurePort=0, ipcPort=42061, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x9705d764108db4e1 with lease ID 0x4f0c6cd0eac05f9f: Processing first storage report for DS-8481b83f-091b-4b2e-86fa-c7074632def2 from datanode DatanodeRegistration(127.0.0.1:39259, datanodeUuid=3caae417-876c-4922-be6c-aa2097011217, infoPort=39897, infoSecurePort=0, ipcPort=38219, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,654 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x9705d764108db4e1 with lease ID 0x4f0c6cd0eac05f9f: from storage DS-8481b83f-091b-4b2e-86fa-c7074632def2 node DatanodeRegistration(127.0.0.1:39259, datanodeUuid=3caae417-876c-4922-be6c-aa2097011217, infoPort=39897, infoSecurePort=0, ipcPort=38219, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,723 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data5/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,724 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data6/current/BP-532991956-172.17.0.2-1732462051681/current, will proceed with Du for space computation calculation, 2024-11-24T15:27:33,755 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-24T15:27:33,760 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfab2ac113132b47f with lease ID 0x4f0c6cd0eac05fa1: Processing first storage report for DS-7326fdfb-1211-4980-9f90-113141d326d8 from datanode DatanodeRegistration(127.0.0.1:34483, datanodeUuid=a852540f-7401-4edc-a24f-ae014780e5b6, infoPort=43973, infoSecurePort=0, ipcPort=40197, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,760 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfab2ac113132b47f with lease ID 0x4f0c6cd0eac05fa1: from storage DS-7326fdfb-1211-4980-9f90-113141d326d8 node DatanodeRegistration(127.0.0.1:34483, datanodeUuid=a852540f-7401-4edc-a24f-ae014780e5b6, infoPort=43973, infoSecurePort=0, ipcPort=40197, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,761 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xfab2ac113132b47f with lease ID 0x4f0c6cd0eac05fa1: Processing first storage report for DS-d8b4203e-1ead-4a56-979b-eedc2371c514 from datanode DatanodeRegistration(127.0.0.1:34483, datanodeUuid=a852540f-7401-4edc-a24f-ae014780e5b6, infoPort=43973, infoSecurePort=0, ipcPort=40197, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681) 2024-11-24T15:27:33,761 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xfab2ac113132b47f with lease ID 0x4f0c6cd0eac05fa1: from storage DS-d8b4203e-1ead-4a56-979b-eedc2371c514 node DatanodeRegistration(127.0.0.1:34483, datanodeUuid=a852540f-7401-4edc-a24f-ae014780e5b6, infoPort=43973, infoSecurePort=0, ipcPort=40197, storageInfo=lv=-57;cid=testClusterID;nsid=1545304655;c=1732462051681), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-24T15:27:33,965 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857 2024-11-24T15:27:34,053 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/zookeeper_0, clientPort=64743, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-24T15:27:34,065 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=64743 2024-11-24T15:27:34,078 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:34,081 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:34,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741825_1001 (size=7) 2024-11-24T15:27:34,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741825_1001 (size=7) 2024-11-24T15:27:34,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741825_1001 (size=7) 2024-11-24T15:27:34,716 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f with version=8 2024-11-24T15:27:34,716 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/hbase-staging 2024-11-24T15:27:35,027 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c99348da3e93:0 server-side Connection retries=45 2024-11-24T15:27:35,038 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,039 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,044 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T15:27:35,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,044 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T15:27:35,185 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-24T15:27:35,247 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-24T15:27:35,256 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-24T15:27:35,260 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T15:27:35,288 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 17507 (auto-detected) 2024-11-24T15:27:35,289 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-24T15:27:35,308 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40561 2024-11-24T15:27:35,328 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40561 connecting to ZooKeeper ensemble=127.0.0.1:64743 2024-11-24T15:27:35,366 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:405610x0, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T15:27:35,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40561-0x1014a96f51f0000 connected 2024-11-24T15:27:35,396 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,399 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,409 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:27:35,413 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f, hbase.cluster.distributed=false 2024-11-24T15:27:35,435 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T15:27:35,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40561 2024-11-24T15:27:35,439 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40561 2024-11-24T15:27:35,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40561 2024-11-24T15:27:35,440 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40561 2024-11-24T15:27:35,441 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40561 2024-11-24T15:27:35,563 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c99348da3e93:0 server-side Connection retries=45 2024-11-24T15:27:35,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T15:27:35,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T15:27:35,568 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T15:27:35,571 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T15:27:35,571 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:38975 2024-11-24T15:27:35,573 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:38975 connecting to ZooKeeper ensemble=127.0.0.1:64743 2024-11-24T15:27:35,574 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,579 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,587 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:389750x0, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T15:27:35,588 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:38975-0x1014a96f51f0001 connected 2024-11-24T15:27:35,588 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:27:35,594 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T15:27:35,605 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T15:27:35,608 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T15:27:35,613 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T15:27:35,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38975 2024-11-24T15:27:35,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38975 2024-11-24T15:27:35,615 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38975 2024-11-24T15:27:35,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38975 2024-11-24T15:27:35,616 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38975 2024-11-24T15:27:35,634 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c99348da3e93:0 server-side Connection retries=45 2024-11-24T15:27:35,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,635 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T15:27:35,635 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,636 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T15:27:35,636 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T15:27:35,636 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T15:27:35,637 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33951 2024-11-24T15:27:35,639 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33951 connecting to ZooKeeper ensemble=127.0.0.1:64743 2024-11-24T15:27:35,640 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,643 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,648 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:339510x0, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T15:27:35,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33951-0x1014a96f51f0002 connected 2024-11-24T15:27:35,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:27:35,650 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T15:27:35,652 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T15:27:35,654 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T15:27:35,655 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T15:27:35,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-24T15:27:35,656 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33951 2024-11-24T15:27:35,660 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33951 2024-11-24T15:27:35,661 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-24T15:27:35,662 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33951 2024-11-24T15:27:35,679 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c99348da3e93:0 server-side Connection retries=45 2024-11-24T15:27:35,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,679 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,679 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-24T15:27:35,680 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-24T15:27:35,680 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-24T15:27:35,680 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-24T15:27:35,680 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-24T15:27:35,681 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:41447 2024-11-24T15:27:35,682 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41447 connecting to ZooKeeper ensemble=127.0.0.1:64743 2024-11-24T15:27:35,683 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,686 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:414470x0, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-24T15:27:35,692 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41447-0x1014a96f51f0003 connected 2024-11-24T15:27:35,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:27:35,692 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-24T15:27:35,693 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-24T15:27:35,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-24T15:27:35,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-24T15:27:35,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41447 2024-11-24T15:27:35,697 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41447 2024-11-24T15:27:35,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41447 2024-11-24T15:27:35,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41447 2024-11-24T15:27:35,702 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41447 2024-11-24T15:27:35,718 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c99348da3e93:40561 2024-11-24T15:27:35,720 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c99348da3e93,40561,1732462054824 2024-11-24T15:27:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,729 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,732 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c99348da3e93,40561,1732462054824 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,754 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-24T15:27:35,755 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c99348da3e93,40561,1732462054824 from backup master directory 2024-11-24T15:27:35,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c99348da3e93,40561,1732462054824 2024-11-24T15:27:35,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-24T15:27:35,759 WARN [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T15:27:35,759 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c99348da3e93,40561,1732462054824 2024-11-24T15:27:35,761 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-24T15:27:35,763 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-24T15:27:35,820 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/hbase.id] with ID: 357eacd2-e660-4f26-ba8c-7141b8360e78 2024-11-24T15:27:35,820 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/.tmp/hbase.id 2024-11-24T15:27:35,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741826_1002 (size=42) 2024-11-24T15:27:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741826_1002 (size=42) 2024-11-24T15:27:35,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741826_1002 (size=42) 2024-11-24T15:27:35,835 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/.tmp/hbase.id]:[hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/hbase.id] 2024-11-24T15:27:35,878 INFO [master/c99348da3e93:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-24T15:27:35,884 INFO [master/c99348da3e93:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-24T15:27:35,906 INFO [master/c99348da3e93:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 20ms. 2024-11-24T15:27:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:35,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741827_1003 (size=196) 2024-11-24T15:27:35,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741827_1003 (size=196) 2024-11-24T15:27:35,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741827_1003 (size=196) 2024-11-24T15:27:35,947 INFO [master/c99348da3e93:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T15:27:35,949 INFO [master/c99348da3e93:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-24T15:27:35,957 INFO [master/c99348da3e93:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:35,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741828_1004 (size=1189) 2024-11-24T15:27:35,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741828_1004 (size=1189) 2024-11-24T15:27:35,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741828_1004 (size=1189) 2024-11-24T15:27:36,008 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store 2024-11-24T15:27:36,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741829_1005 (size=34) 2024-11-24T15:27:36,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741829_1005 (size=34) 2024-11-24T15:27:36,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741829_1005 (size=34) 2024-11-24T15:27:36,033 INFO [master/c99348da3e93:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-24T15:27:36,036 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:36,037 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T15:27:36,038 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:27:36,038 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:27:36,039 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T15:27:36,039 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:27:36,039 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:27:36,041 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732462056037Disabling compacts and flushes for region at 1732462056037Disabling writes for close at 1732462056039 (+2 ms)Writing region close event to WAL at 1732462056039Closed at 1732462056039 2024-11-24T15:27:36,042 WARN [master/c99348da3e93:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/.initializing 2024-11-24T15:27:36,043 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/WALs/c99348da3e93,40561,1732462054824 2024-11-24T15:27:36,051 INFO [master/c99348da3e93:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:36,066 INFO [master/c99348da3e93:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c99348da3e93%2C40561%2C1732462054824, suffix=, logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/WALs/c99348da3e93,40561,1732462054824, archiveDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/oldWALs, maxLogs=10 2024-11-24T15:27:36,096 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/WALs/c99348da3e93,40561,1732462054824/c99348da3e93%2C40561%2C1732462054824.1732462056070, exclude list is [], retry=0 2024-11-24T15:27:36,115 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:36,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:36,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:36,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:36,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-24T15:27:36,159 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-11-24T15:27:36,220 INFO [master/c99348da3e93:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/WALs/c99348da3e93,40561,1732462054824/c99348da3e93%2C40561%2C1732462054824.1732462056070 2024-11-24T15:27:36,221 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:36,221 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:36,222 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:36,225 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,226 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,260 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,286 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-24T15:27:36,290 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:36,293 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,296 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-24T15:27:36,296 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:36,297 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,300 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-24T15:27:36,300 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:36,301 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,303 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-24T15:27:36,303 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,304 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:36,304 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,307 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,309 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,314 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,315 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,318 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T15:27:36,322 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-24T15:27:36,326 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:36,327 INFO [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66985701, jitterRate=-0.0018352717161178589}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T15:27:36,333 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732462056237Initializing all the Stores at 1732462056239 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462056240 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462056241 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462056241Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462056241Cleaning up temporary data from old regions at 1732462056315 (+74 ms)Region opened successfully at 1732462056333 (+18 ms) 2024-11-24T15:27:36,334 INFO [master/c99348da3e93:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-24T15:27:36,371 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c9e3c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c99348da3e93/172.17.0.2:0 2024-11-24T15:27:36,407 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-24T15:27:36,420 INFO [master/c99348da3e93:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-24T15:27:36,420 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-24T15:27:36,424 INFO [master/c99348da3e93:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-24T15:27:36,426 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-24T15:27:36,432 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-11-24T15:27:36,432 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-24T15:27:36,462 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-24T15:27:36,469 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-24T15:27:36,473 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-24T15:27:36,475 INFO [master/c99348da3e93:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-24T15:27:36,476 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-24T15:27:36,477 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-24T15:27:36,480 INFO [master/c99348da3e93:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-24T15:27:36,483 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-24T15:27:36,484 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-24T15:27:36,485 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-24T15:27:36,486 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-24T15:27:36,503 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-24T15:27:36,506 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,513 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c99348da3e93,40561,1732462054824, sessionid=0x1014a96f51f0000, setting cluster-up flag (Was=false) 2024-11-24T15:27:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,531 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-24T15:27:36,533 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c99348da3e93,40561,1732462054824 2024-11-24T15:27:36,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:36,544 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-24T15:27:36,546 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c99348da3e93,40561,1732462054824 2024-11-24T15:27:36,551 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-24T15:27:36,606 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(746): ClusterId : 357eacd2-e660-4f26-ba8c-7141b8360e78 2024-11-24T15:27:36,606 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(746): ClusterId : 357eacd2-e660-4f26-ba8c-7141b8360e78 2024-11-24T15:27:36,606 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(746): ClusterId : 357eacd2-e660-4f26-ba8c-7141b8360e78 2024-11-24T15:27:36,609 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T15:27:36,609 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T15:27:36,609 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-24T15:27:36,616 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T15:27:36,616 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T15:27:36,616 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-24T15:27:36,616 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T15:27:36,616 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T15:27:36,616 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-24T15:27:36,620 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T15:27:36,620 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T15:27:36,620 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-24T15:27:36,621 DEBUG [RS:1;c99348da3e93:33951 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30c5d15c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c99348da3e93/172.17.0.2:0 2024-11-24T15:27:36,621 DEBUG [RS:0;c99348da3e93:38975 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c0ac255, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c99348da3e93/172.17.0.2:0 2024-11-24T15:27:36,621 DEBUG [RS:2;c99348da3e93:41447 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e753091, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c99348da3e93/172.17.0.2:0 2024-11-24T15:27:36,628 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-24T15:27:36,638 INFO [master/c99348da3e93:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-24T15:27:36,639 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c99348da3e93:38975 2024-11-24T15:27:36,640 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c99348da3e93:41447 2024-11-24T15:27:36,641 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c99348da3e93:33951 2024-11-24T15:27:36,643 INFO [RS:0;c99348da3e93:38975 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T15:27:36,643 INFO [RS:2;c99348da3e93:41447 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T15:27:36,643 INFO [RS:1;c99348da3e93:33951 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-24T15:27:36,643 INFO [RS:0;c99348da3e93:38975 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T15:27:36,643 INFO [RS:2;c99348da3e93:41447 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T15:27:36,643 INFO [RS:1;c99348da3e93:33951 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-24T15:27:36,643 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T15:27:36,643 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T15:27:36,643 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-24T15:27:36,644 INFO [master/c99348da3e93:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-24T15:27:36,646 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(2659): reportForDuty to master=c99348da3e93,40561,1732462054824 with port=38975, startcode=1732462055523 2024-11-24T15:27:36,646 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(2659): reportForDuty to master=c99348da3e93,40561,1732462054824 with port=33951, startcode=1732462055634 2024-11-24T15:27:36,646 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(2659): reportForDuty to master=c99348da3e93,40561,1732462054824 with port=41447, startcode=1732462055679 2024-11-24T15:27:36,650 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c99348da3e93,40561,1732462054824 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-24T15:27:36,657 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c99348da3e93:0, corePoolSize=5, maxPoolSize=5 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c99348da3e93:0, corePoolSize=5, maxPoolSize=5 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c99348da3e93:0, corePoolSize=5, maxPoolSize=5 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c99348da3e93:0, corePoolSize=5, maxPoolSize=5 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c99348da3e93:0, corePoolSize=10, maxPoolSize=10 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,658 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c99348da3e93:0, corePoolSize=2, maxPoolSize=2 2024-11-24T15:27:36,659 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,660 DEBUG [RS:2;c99348da3e93:41447 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T15:27:36,660 DEBUG [RS:0;c99348da3e93:38975 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T15:27:36,660 DEBUG [RS:1;c99348da3e93:33951 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-24T15:27:36,665 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732462086665 2024-11-24T15:27:36,667 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-24T15:27:36,668 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-24T15:27:36,671 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T15:27:36,671 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-24T15:27:36,672 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-24T15:27:36,672 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-24T15:27:36,673 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-24T15:27:36,673 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-24T15:27:36,674 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,680 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-24T15:27:36,680 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,681 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-24T15:27:36,682 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-24T15:27:36,684 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-24T15:27:36,680 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-24T15:27:36,685 INFO [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-24T15:27:36,688 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.large.0-1732462056686,5,FailOnTimeoutGroup] 2024-11-24T15:27:36,689 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.small.0-1732462056688,5,FailOnTimeoutGroup] 2024-11-24T15:27:36,689 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,689 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-24T15:27:36,690 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,690 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741831_1007 (size=1321) 2024-11-24T15:27:36,723 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52301, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T15:27:36,723 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59089, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T15:27:36,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741831_1007 (size=1321) 2024-11-24T15:27:36,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741831_1007 (size=1321) 2024-11-24T15:27:36,724 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56133, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-24T15:27:36,727 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-24T15:27:36,727 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f 2024-11-24T15:27:36,731 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,733 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(517): Registering regionserver=c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741832_1008 (size=32) 2024-11-24T15:27:36,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741832_1008 (size=32) 2024-11-24T15:27:36,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741832_1008 (size=32) 2024-11-24T15:27:36,746 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:36,747 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,747 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(517): Registering regionserver=c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,749 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T15:27:36,751 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f 2024-11-24T15:27:36,751 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44579 2024-11-24T15:27:36,751 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T15:27:36,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,752 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f 2024-11-24T15:27:36,752 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40561 {}] master.ServerManager(517): Registering regionserver=c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,752 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44579 2024-11-24T15:27:36,752 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T15:27:36,754 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T15:27:36,754 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,755 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:36,756 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T15:27:36,756 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f 2024-11-24T15:27:36,756 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44579 2024-11-24T15:27:36,756 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-24T15:27:36,757 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T15:27:36,758 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T15:27:36,758 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:36,759 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T15:27:36,762 DEBUG [RS:1;c99348da3e93:33951 {}] zookeeper.ZKUtil(111): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,762 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T15:27:36,762 WARN [RS:1;c99348da3e93:33951 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T15:27:36,762 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,762 INFO [RS:1;c99348da3e93:33951 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:36,763 DEBUG [RS:0;c99348da3e93:38975 {}] zookeeper.ZKUtil(111): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,763 DEBUG [RS:2;c99348da3e93:41447 {}] zookeeper.ZKUtil(111): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,763 WARN [RS:0;c99348da3e93:38975 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T15:27:36,763 WARN [RS:2;c99348da3e93:41447 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-24T15:27:36,763 INFO [RS:0;c99348da3e93:38975 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:36,763 INFO [RS:2;c99348da3e93:41447 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:36,763 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c99348da3e93,38975,1732462055523] 2024-11-24T15:27:36,763 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c99348da3e93,41447,1732462055679] 2024-11-24T15:27:36,763 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,764 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c99348da3e93,33951,1732462055634] 2024-11-24T15:27:36,763 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:36,765 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T15:27:36,767 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T15:27:36,767 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:36,768 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:36,768 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T15:27:36,770 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740 2024-11-24T15:27:36,771 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740 2024-11-24T15:27:36,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T15:27:36,774 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T15:27:36,776 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T15:27:36,779 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T15:27:36,783 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:36,784 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64762046, jitterRate=-0.03497031331062317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T15:27:36,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732462056746Initializing all the Stores at 1732462056748 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462056748Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462056748Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462056748Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462056748Cleaning up temporary data from old regions at 1732462056774 (+26 ms)Region opened successfully at 1732462056787 (+13 ms) 2024-11-24T15:27:36,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T15:27:36,788 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T15:27:36,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T15:27:36,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T15:27:36,788 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T15:27:36,789 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T15:27:36,790 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732462056788Disabling compacts and flushes for region at 1732462056788Disabling writes for close at 1732462056788Writing region close event to WAL at 1732462056789 (+1 ms)Closed at 1732462056789 2024-11-24T15:27:36,793 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T15:27:36,793 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-24T15:27:36,797 INFO [RS:2;c99348da3e93:41447 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T15:27:36,797 INFO [RS:0;c99348da3e93:38975 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T15:27:36,797 INFO [RS:1;c99348da3e93:33951 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-24T15:27:36,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-24T15:27:36,811 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T15:27:36,814 INFO [RS:0;c99348da3e93:38975 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T15:27:36,814 INFO [RS:1;c99348da3e93:33951 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T15:27:36,814 INFO [RS:2;c99348da3e93:41447 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-24T15:27:36,814 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-24T15:27:36,820 INFO [RS:2;c99348da3e93:41447 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T15:27:36,820 INFO [RS:1;c99348da3e93:33951 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T15:27:36,820 INFO [RS:0;c99348da3e93:38975 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T15:27:36,820 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,820 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,820 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,821 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T15:27:36,821 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T15:27:36,822 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-24T15:27:36,828 INFO [RS:1;c99348da3e93:33951 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T15:27:36,828 INFO [RS:2;c99348da3e93:41447 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T15:27:36,829 INFO [RS:0;c99348da3e93:38975 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-24T15:27:36,830 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,830 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,830 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,830 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,830 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,830 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,830 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,830 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,830 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0, corePoolSize=2, maxPoolSize=2 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0, corePoolSize=2, maxPoolSize=2 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0, corePoolSize=2, maxPoolSize=2 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,831 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c99348da3e93:0, corePoolSize=1, maxPoolSize=1 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,832 DEBUG [RS:2;c99348da3e93:41447 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,832 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,832 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,833 DEBUG [RS:1;c99348da3e93:33951 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,833 DEBUG [RS:0;c99348da3e93:38975 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c99348da3e93:0, corePoolSize=3, maxPoolSize=3 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,38975,1732462055523-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,33951,1732462055634-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T15:27:36,834 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,41447,1732462055679-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T15:27:36,854 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T15:27:36,854 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T15:27:36,856 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,33951,1732462055634-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,856 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,38975,1732462055523-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,856 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,856 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,857 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-24T15:27:36,857 INFO [RS:1;c99348da3e93:33951 {}] regionserver.Replication(171): c99348da3e93,33951,1732462055634 started 2024-11-24T15:27:36,857 INFO [RS:0;c99348da3e93:38975 {}] regionserver.Replication(171): c99348da3e93,38975,1732462055523 started 2024-11-24T15:27:36,857 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,41447,1732462055679-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,857 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,857 INFO [RS:2;c99348da3e93:41447 {}] regionserver.Replication(171): c99348da3e93,41447,1732462055679 started 2024-11-24T15:27:36,875 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,875 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1482): Serving as c99348da3e93,33951,1732462055634, RpcServer on c99348da3e93/172.17.0.2:33951, sessionid=0x1014a96f51f0002 2024-11-24T15:27:36,876 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T15:27:36,876 DEBUG [RS:1;c99348da3e93:33951 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,877 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,33951,1732462055634' 2024-11-24T15:27:36,877 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T15:27:36,877 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,877 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1482): Serving as c99348da3e93,41447,1732462055679, RpcServer on c99348da3e93/172.17.0.2:41447, sessionid=0x1014a96f51f0003 2024-11-24T15:27:36,877 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T15:27:36,878 DEBUG [RS:2;c99348da3e93:41447 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,878 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,41447,1732462055679' 2024-11-24T15:27:36,878 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T15:27:36,878 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T15:27:36,878 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T15:27:36,878 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T15:27:36,879 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T15:27:36,879 DEBUG [RS:1;c99348da3e93:33951 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c99348da3e93,33951,1732462055634 2024-11-24T15:27:36,879 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,33951,1732462055634' 2024-11-24T15:27:36,879 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T15:27:36,879 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T15:27:36,879 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T15:27:36,879 DEBUG [RS:2;c99348da3e93:41447 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c99348da3e93,41447,1732462055679 2024-11-24T15:27:36,879 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,41447,1732462055679' 2024-11-24T15:27:36,879 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T15:27:36,879 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T15:27:36,880 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T15:27:36,880 DEBUG [RS:1;c99348da3e93:33951 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T15:27:36,880 INFO [RS:1;c99348da3e93:33951 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T15:27:36,880 INFO [RS:1;c99348da3e93:33951 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T15:27:36,880 DEBUG [RS:2;c99348da3e93:41447 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T15:27:36,880 INFO [RS:2;c99348da3e93:41447 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T15:27:36,881 INFO [RS:2;c99348da3e93:41447 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T15:27:36,882 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:36,882 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1482): Serving as c99348da3e93,38975,1732462055523, RpcServer on c99348da3e93/172.17.0.2:38975, sessionid=0x1014a96f51f0001 2024-11-24T15:27:36,882 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-24T15:27:36,883 DEBUG [RS:0;c99348da3e93:38975 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,883 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,38975,1732462055523' 2024-11-24T15:27:36,883 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-24T15:27:36,883 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-24T15:27:36,884 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-24T15:27:36,884 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-24T15:27:36,884 DEBUG [RS:0;c99348da3e93:38975 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c99348da3e93,38975,1732462055523 2024-11-24T15:27:36,884 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c99348da3e93,38975,1732462055523' 2024-11-24T15:27:36,884 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-24T15:27:36,885 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-24T15:27:36,885 DEBUG [RS:0;c99348da3e93:38975 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-24T15:27:36,885 INFO [RS:0;c99348da3e93:38975 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-24T15:27:36,886 INFO [RS:0;c99348da3e93:38975 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-24T15:27:36,965 WARN [c99348da3e93:40561 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-24T15:27:36,986 INFO [RS:1;c99348da3e93:33951 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:36,986 INFO [RS:2;c99348da3e93:41447 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:36,986 INFO [RS:0;c99348da3e93:38975 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:36,990 INFO [RS:0;c99348da3e93:38975 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c99348da3e93%2C38975%2C1732462055523, suffix=, logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,38975,1732462055523, archiveDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs, maxLogs=32 2024-11-24T15:27:36,990 INFO [RS:2;c99348da3e93:41447 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c99348da3e93%2C41447%2C1732462055679, suffix=, logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679, archiveDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs, maxLogs=32 2024-11-24T15:27:36,990 INFO [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c99348da3e93%2C33951%2C1732462055634, suffix=, logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634, archiveDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs, maxLogs=32 2024-11-24T15:27:37,009 DEBUG [RS:0;c99348da3e93:38975 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,38975,1732462055523/c99348da3e93%2C38975%2C1732462055523.1732462056994, exclude list is [], retry=0 2024-11-24T15:27:37,009 DEBUG [RS:2;c99348da3e93:41447 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679/c99348da3e93%2C41447%2C1732462055679.1732462056995, exclude list is [], retry=0 2024-11-24T15:27:37,010 DEBUG [RS:1;c99348da3e93:33951 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634/c99348da3e93%2C33951%2C1732462055634.1732462056995, exclude list is [], retry=0 2024-11-24T15:27:37,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:37,015 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:37,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:37,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:37,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:37,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:37,017 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:37,018 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:37,057 INFO [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634/c99348da3e93%2C33951%2C1732462055634.1732462056995 2024-11-24T15:27:37,058 DEBUG [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:27:37,065 INFO [RS:0;c99348da3e93:38975 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,38975,1732462055523/c99348da3e93%2C38975%2C1732462055523.1732462056994 2024-11-24T15:27:37,066 DEBUG [RS:0;c99348da3e93:38975 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:37,076 INFO [RS:2;c99348da3e93:41447 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679/c99348da3e93%2C41447%2C1732462055679.1732462056995 2024-11-24T15:27:37,076 DEBUG [RS:2;c99348da3e93:41447 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:37,218 DEBUG [c99348da3e93:40561 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-24T15:27:37,226 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(204): Hosts are {c99348da3e93=0} racks are {/default-rack=0} 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T15:27:37,232 INFO [c99348da3e93:40561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T15:27:37,232 INFO [c99348da3e93:40561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T15:27:37,232 INFO [c99348da3e93:40561 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T15:27:37,232 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T15:27:37,240 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c99348da3e93,33951,1732462055634 2024-11-24T15:27:37,245 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c99348da3e93,33951,1732462055634, state=OPENING 2024-11-24T15:27:37,250 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-24T15:27:37,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:37,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:37,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:37,253 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:27:37,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,254 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,255 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,256 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-24T15:27:37,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c99348da3e93,33951,1732462055634}] 2024-11-24T15:27:37,433 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T15:27:37,435 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57967, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T15:27:37,447 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-24T15:27:37,448 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:37,448 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-24T15:27:37,451 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c99348da3e93%2C33951%2C1732462055634.meta, suffix=.meta, logDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634, archiveDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs, maxLogs=32 2024-11-24T15:27:37,468 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634/c99348da3e93%2C33951%2C1732462055634.meta.1732462057453.meta, exclude list is [], retry=0 2024-11-24T15:27:37,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:37,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:37,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:37,503 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634/c99348da3e93%2C33951%2C1732462055634.meta.1732462057453.meta 2024-11-24T15:27:37,504 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:37,504 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:37,506 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-24T15:27:37,509 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-24T15:27:37,513 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-24T15:27:37,518 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-24T15:27:37,518 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:37,519 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-24T15:27:37,519 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-24T15:27:37,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-24T15:27:37,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-24T15:27:37,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:37,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:37,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-24T15:27:37,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-24T15:27:37,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:37,527 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:37,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-24T15:27:37,529 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-24T15:27:37,529 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:37,529 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:37,530 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-24T15:27:37,531 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-24T15:27:37,531 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:37,532 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-24T15:27:37,532 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-24T15:27:37,533 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740 2024-11-24T15:27:37,536 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740 2024-11-24T15:27:37,538 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-24T15:27:37,538 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-24T15:27:37,539 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-24T15:27:37,541 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-24T15:27:37,543 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64754765, jitterRate=-0.035078808665275574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-24T15:27:37,544 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-24T15:27:37,545 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732462057519Writing region info on filesystem at 1732462057520 (+1 ms)Initializing all the Stores at 1732462057522 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462057522Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462057522Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462057522Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732462057522Cleaning up temporary data from old regions at 1732462057538 (+16 ms)Running coprocessor post-open hooks at 1732462057544 (+6 ms)Region opened successfully at 1732462057545 (+1 ms) 2024-11-24T15:27:37,553 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732462057424 2024-11-24T15:27:37,565 DEBUG [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-24T15:27:37,565 INFO [RS_OPEN_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-24T15:27:37,567 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c99348da3e93,33951,1732462055634 2024-11-24T15:27:37,569 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c99348da3e93,33951,1732462055634, state=OPEN 2024-11-24T15:27:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T15:27:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T15:27:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T15:27:37,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-24T15:27:37,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,572 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-24T15:27:37,572 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c99348da3e93,33951,1732462055634 2024-11-24T15:27:37,578 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-24T15:27:37,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c99348da3e93,33951,1732462055634 in 315 msec 2024-11-24T15:27:37,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-24T15:27:37,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 779 msec 2024-11-24T15:27:37,586 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-24T15:27:37,586 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-24T15:27:37,638 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T15:27:37,640 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c99348da3e93,33951,1732462055634, seqNum=-1] 2024-11-24T15:27:37,669 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T15:27:37,672 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55497, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T15:27:37,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1110 sec 2024-11-24T15:27:37,691 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732462057690, completionTime=-1 2024-11-24T15:27:37,693 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-24T15:27:37,693 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-24T15:27:37,718 INFO [master/c99348da3e93:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-24T15:27:37,718 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732462117718 2024-11-24T15:27:37,718 INFO [master/c99348da3e93:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732462177718 2024-11-24T15:27:37,718 INFO [master/c99348da3e93:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 25 msec 2024-11-24T15:27:37,719 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-24T15:27:37,726 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,726 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,727 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,728 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c99348da3e93:40561, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,728 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,729 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,735 DEBUG [master/c99348da3e93:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-24T15:27:37,757 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.997sec 2024-11-24T15:27:37,758 INFO [master/c99348da3e93:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-24T15:27:37,760 INFO [master/c99348da3e93:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-24T15:27:37,761 INFO [master/c99348da3e93:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-24T15:27:37,762 INFO [master/c99348da3e93:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-24T15:27:37,762 INFO [master/c99348da3e93:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-24T15:27:37,763 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-24T15:27:37,763 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-24T15:27:37,768 DEBUG [master/c99348da3e93:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-24T15:27:37,769 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-24T15:27:37,769 INFO [master/c99348da3e93:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c99348da3e93,40561,1732462054824-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:37,830 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51245b74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T15:27:37,831 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c99348da3e93,40561,-1 for getting cluster id 2024-11-24T15:27:37,833 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-24T15:27:37,842 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '357eacd2-e660-4f26-ba8c-7141b8360e78' 2024-11-24T15:27:37,844 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-24T15:27:37,844 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "357eacd2-e660-4f26-ba8c-7141b8360e78" 2024-11-24T15:27:37,845 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@272a2ec5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T15:27:37,845 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c99348da3e93,40561,-1] 2024-11-24T15:27:37,847 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-24T15:27:37,848 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:27:37,850 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47598, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-24T15:27:37,852 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4010c1c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-24T15:27:37,853 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-24T15:27:37,860 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c99348da3e93,33951,1732462055634, seqNum=-1] 2024-11-24T15:27:37,860 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T15:27:37,863 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44878, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T15:27:37,883 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c99348da3e93,40561,1732462054824 2024-11-24T15:27:37,883 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44579/hbase 2024-11-24T15:27:37,899 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=607, MaxFileDescriptor=1048576, SystemLoadAverage=247, ProcessCount=11, AvailableMemoryMB=6827 2024-11-24T15:27:37,917 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:37,921 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:37,921 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:37,926 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-80474863, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-80474863, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:37,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-80474863/hregion-80474863.1732462057927, exclude list is [], retry=0 2024-11-24T15:27:37,946 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:37,946 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:37,947 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:37,977 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-80474863/hregion-80474863.1732462057927 2024-11-24T15:27:37,977 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:37,977 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => d9d5a9baf30d925be3a917c9ae6ed405, NAME => 'testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741838_1014 (size=64) 2024-11-24T15:27:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741838_1014 (size=64) 2024-11-24T15:27:37,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741838_1014 (size=64) 2024-11-24T15:27:37,993 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:37,995 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:37,998 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9d5a9baf30d925be3a917c9ae6ed405 columnFamilyName a 2024-11-24T15:27:37,998 DEBUG [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:37,999 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] regionserver.HStore(327): Store=d9d5a9baf30d925be3a917c9ae6ed405/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:37,999 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,001 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,002 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,002 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,005 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,008 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:38,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9d5a9baf30d925be3a917c9ae6ed405; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61365854, jitterRate=-0.08557751774787903}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:27:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9d5a9baf30d925be3a917c9ae6ed405: Writing region info on filesystem at 1732462057993Initializing all the Stores at 1732462057995 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462057995Cleaning up temporary data from old regions at 1732462058002 (+7 ms)Region opened successfully at 1732462058010 (+8 ms) 2024-11-24T15:27:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing d9d5a9baf30d925be3a917c9ae6ed405, disabling compactions & flushes 2024-11-24T15:27:38,010 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. 2024-11-24T15:27:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. 2024-11-24T15:27:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. after waiting 0 ms 2024-11-24T15:27:38,010 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. 2024-11-24T15:27:38,011 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. 2024-11-24T15:27:38,011 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for d9d5a9baf30d925be3a917c9ae6ed405: Waiting for close lock at 1732462058010Disabling compacts and flushes for region at 1732462058010Disabling writes for close at 1732462058010Writing region close event to WAL at 1732462058011 (+1 ms)Closed at 1732462058011 2024-11-24T15:27:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741837_1013 (size=175) 2024-11-24T15:27:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741837_1013 (size=175) 2024-11-24T15:27:38,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741837_1013 (size=175) 2024-11-24T15:27:38,022 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:38,022 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-80474863:(num 1732462057927) 2024-11-24T15:27:38,024 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T15:27:38,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741839_1015 (size=469) 2024-11-24T15:27:38,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741839_1015 (size=469) 2024-11-24T15:27:38,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741839_1015 (size=469) 2024-11-24T15:27:38,070 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T15:27:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741840_1016 (size=322) 2024-11-24T15:27:38,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741840_1016 (size=322) 2024-11-24T15:27:38,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741840_1016 (size=322) 2024-11-24T15:27:38,137 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1, size=469 (469bytes) 2024-11-24T15:27:38,138 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-24T15:27:38,138 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-24T15:27:38,139 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1 2024-11-24T15:27:38,144 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1 after 3ms 2024-11-24T15:27:38,177 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,178 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1 took 42ms 2024-11-24T15:27:38,182 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1 so closing down 2024-11-24T15:27:38,182 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:38,185 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T15:27:38,211 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T15:27:38,212 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:38,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741841_1017 (size=469) 2024-11-24T15:27:38,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741841_1017 (size=469) 2024-11-24T15:27:38,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741841_1017 (size=469) 2024-11-24T15:27:38,225 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:38,227 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002 2024-11-24T15:27:38,233 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T15:27:38,233 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1, journal: Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1, size=469 (469bytes) at 1732462058138Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1 so closing down at 1732462058182 (+44 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp at 1732462058211 (+29 ms)3 split writer threads finished at 1732462058212 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732462058225 (+13 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002 at 1732462058227 (+2 ms)Processed 2 edits across 1 Regions in 49 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732462058233 (+6 ms) 2024-11-24T15:27:38,252 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2, size=322 (322bytes) 2024-11-24T15:27:38,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2 2024-11-24T15:27:38,252 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2 after 0ms 2024-11-24T15:27:38,281 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,281 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2 took 30ms 2024-11-24T15:27:38,284 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2 so closing down 2024-11-24T15:27:38,284 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:38,287 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T15:27:38,313 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T15:27:38,313 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741842_1018 (size=322) 2024-11-24T15:27:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741842_1018 (size=322) 2024-11-24T15:27:38,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741842_1018 (size=322) 2024-11-24T15:27:38,324 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:38,355 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,357 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-11-24T15:27:38,359 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T15:27:38,359 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2, journal: Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2, size=322 (322bytes) at 1732462058252Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2 so closing down at 1732462058284 (+32 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002-wal-2.temp at 1732462058313 (+29 ms)3 split writer threads finished at 1732462058313Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732462058324 (+11 ms)Processed 1 edits across 1 Regions in 77 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732462058359 (+35 ms) 2024-11-24T15:27:38,359 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:38,362 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:38,376 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal.1732462058363, exclude list is [], retry=0 2024-11-24T15:27:38,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:38,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:38,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:38,408 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal.1732462058363 2024-11-24T15:27:38,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:38,409 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => d9d5a9baf30d925be3a917c9ae6ed405, NAME => 'testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:38,409 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:38,409 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,409 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,412 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,413 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d9d5a9baf30d925be3a917c9ae6ed405 columnFamilyName a 2024-11-24T15:27:38,413 DEBUG [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:38,414 INFO [StoreOpener-d9d5a9baf30d925be3a917c9ae6ed405-1 {}] regionserver.HStore(327): Store=d9d5a9baf30d925be3a917c9ae6ed405/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:38,414 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,415 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,419 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002 2024-11-24T15:27:38,449 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,456 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002 2024-11-24T15:27:38,459 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing d9d5a9baf30d925be3a917c9ae6ed405 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T15:27:38,509 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/.tmp/a/5a93ff0d6805474ab12808518343908c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732462058022/Put/seqid=0 2024-11-24T15:27:38,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741844_1020 (size=5170) 2024-11-24T15:27:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741844_1020 (size=5170) 2024-11-24T15:27:38,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741844_1020 (size=5170) 2024-11-24T15:27:38,524 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/.tmp/a/5a93ff0d6805474ab12808518343908c 2024-11-24T15:27:38,570 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/.tmp/a/5a93ff0d6805474ab12808518343908c as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/a/5a93ff0d6805474ab12808518343908c 2024-11-24T15:27:38,579 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/a/5a93ff0d6805474ab12808518343908c, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T15:27:38,586 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d9d5a9baf30d925be3a917c9ae6ed405 in 125ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T15:27:38,587 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/0000000000000000002 2024-11-24T15:27:38,588 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,588 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for d9d5a9baf30d925be3a917c9ae6ed405 2024-11-24T15:27:38,594 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/d9d5a9baf30d925be3a917c9ae6ed405/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T15:27:38,595 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened d9d5a9baf30d925be3a917c9ae6ed405; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73734726, jitterRate=0.09873303771018982}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:27:38,596 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for d9d5a9baf30d925be3a917c9ae6ed405: Writing region info on filesystem at 1732462058409Initializing all the Stores at 1732462058411 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462058411Obtaining lock to block concurrent updates at 1732462058459 (+48 ms)Preparing flush snapshotting stores in d9d5a9baf30d925be3a917c9ae6ed405 at 1732462058459Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732462058462 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1732462057918.d9d5a9baf30d925be3a917c9ae6ed405. at 1732462058462Flushing d9d5a9baf30d925be3a917c9ae6ed405/a: creating writer at 1732462058463 (+1 ms)Flushing d9d5a9baf30d925be3a917c9ae6ed405/a: appending metadata at 1732462058500 (+37 ms)Flushing d9d5a9baf30d925be3a917c9ae6ed405/a: closing flushed file at 1732462058502 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc93182: reopening flushed file at 1732462058568 (+66 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for d9d5a9baf30d925be3a917c9ae6ed405 in 125ms, sequenceid=2, compaction requested=false; wal=null at 1732462058586 (+18 ms)Cleaning up temporary data from old regions at 1732462058588 (+2 ms)Region opened successfully at 1732462058596 (+8 ms) 2024-11-24T15:27:38,620 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 363) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57582 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:36158 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:36012 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57710 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:55476 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44579/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=699 (was 607) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 247) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6816 (was 6827) 2024-11-24T15:27:38,631 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=6815 2024-11-24T15:27:38,646 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:38,649 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:38,649 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:38,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-90213319, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-90213319, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:38,666 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-90213319/hregion-90213319.1732462058654, exclude list is [], retry=0 2024-11-24T15:27:38,670 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:38,670 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:38,670 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:38,698 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-90213319/hregion-90213319.1732462058654 2024-11-24T15:27:38,698 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:38,698 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 8a3f432e9db720f63dbb9d9a104c33f7, NAME => 'testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:38,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741846_1022 (size=64) 2024-11-24T15:27:38,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741846_1022 (size=64) 2024-11-24T15:27:38,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741846_1022 (size=64) 2024-11-24T15:27:38,710 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:38,712 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,714 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a3f432e9db720f63dbb9d9a104c33f7 columnFamilyName a 2024-11-24T15:27:38,714 DEBUG [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:38,715 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] regionserver.HStore(327): Store=8a3f432e9db720f63dbb9d9a104c33f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:38,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,717 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,717 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,717 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,720 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:38,723 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:38,724 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a3f432e9db720f63dbb9d9a104c33f7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65004003, jitterRate=-0.03136487305164337}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:27:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a3f432e9db720f63dbb9d9a104c33f7: Writing region info on filesystem at 1732462058710Initializing all the Stores at 1732462058711 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462058711Cleaning up temporary data from old regions at 1732462058717 (+6 ms)Region opened successfully at 1732462058724 (+7 ms) 2024-11-24T15:27:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 8a3f432e9db720f63dbb9d9a104c33f7, disabling compactions & flushes 2024-11-24T15:27:38,725 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. 2024-11-24T15:27:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. 2024-11-24T15:27:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. after waiting 0 ms 2024-11-24T15:27:38,725 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. 2024-11-24T15:27:38,725 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. 2024-11-24T15:27:38,726 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 8a3f432e9db720f63dbb9d9a104c33f7: Waiting for close lock at 1732462058725Disabling compacts and flushes for region at 1732462058725Disabling writes for close at 1732462058725Writing region close event to WAL at 1732462058725Closed at 1732462058725 2024-11-24T15:27:38,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741845_1021 (size=175) 2024-11-24T15:27:38,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741845_1021 (size=175) 2024-11-24T15:27:38,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741845_1021 (size=175) 2024-11-24T15:27:38,737 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:38,737 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-90213319:(num 1732462058654) 2024-11-24T15:27:38,738 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T15:27:38,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741847_1023 (size=469) 2024-11-24T15:27:38,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741847_1023 (size=469) 2024-11-24T15:27:38,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741847_1023 (size=469) 2024-11-24T15:27:38,780 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-24T15:27:38,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741848_1024 (size=322) 2024-11-24T15:27:38,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741848_1024 (size=322) 2024-11-24T15:27:38,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741848_1024 (size=322) 2024-11-24T15:27:38,832 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2, size=322 (322bytes) 2024-11-24T15:27:38,832 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2 2024-11-24T15:27:38,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2 after 0ms 2024-11-24T15:27:38,863 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,863 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2 took 32ms 2024-11-24T15:27:38,866 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2 so closing down 2024-11-24T15:27:38,866 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:38,868 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-24T15:27:38,899 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp 2024-11-24T15:27:38,899 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:38,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741849_1025 (size=322) 2024-11-24T15:27:38,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741849_1025 (size=322) 2024-11-24T15:27:38,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741849_1025 (size=322) 2024-11-24T15:27:38,911 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:38,914 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 2024-11-24T15:27:38,914 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-11-24T15:27:38,914 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2, journal: Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2, size=322 (322bytes) at 1732462058832Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2 so closing down at 1732462058866 (+34 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp at 1732462058899 (+33 ms)3 split writer threads finished at 1732462058899Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732462058911 (+12 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 at 1732462058914 (+3 ms)Processed 1 edits across 1 Regions in 50 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1732462058914 2024-11-24T15:27:38,933 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1, size=469 (469bytes) 2024-11-24T15:27:38,933 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1 2024-11-24T15:27:38,934 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1 after 1ms 2024-11-24T15:27:38,968 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:38,968 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1 took 35ms 2024-11-24T15:27:38,972 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1 so closing down 2024-11-24T15:27:38,972 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:38,975 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-24T15:27:39,005 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp 2024-11-24T15:27:39,005 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:39,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741850_1026 (size=469) 2024-11-24T15:27:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741850_1026 (size=469) 2024-11-24T15:27:39,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741850_1026 (size=469) 2024-11-24T15:27:39,016 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:39,047 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:39,051 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002, length=322 2024-11-24T15:27:39,054 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 2024-11-24T15:27:39,054 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-11-24T15:27:39,054 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1, journal: Splitting hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1, size=469 (469bytes) at 1732462058933Finishing writing output for hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1 so closing down at 1732462058972 (+39 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp at 1732462059005 (+33 ms)3 split writer threads finished at 1732462059005Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732462059016 (+11 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 at 1732462059054 (+38 ms)Processed 2 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1732462059054 2024-11-24T15:27:39,054 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:39,057 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:39,071 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal.1732462059057, exclude list is [], retry=0 2024-11-24T15:27:39,074 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:39,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:39,075 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:39,112 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal.1732462059057 2024-11-24T15:27:39,112 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a3f432e9db720f63dbb9d9a104c33f7, NAME => 'testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,113 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,115 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,117 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a3f432e9db720f63dbb9d9a104c33f7 columnFamilyName a 2024-11-24T15:27:39,117 DEBUG [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:39,118 INFO [StoreOpener-8a3f432e9db720f63dbb9d9a104c33f7-1 {}] regionserver.HStore(327): Store=8a3f432e9db720f63dbb9d9a104c33f7/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:39,118 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,119 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,122 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,123 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 2024-11-24T15:27:39,153 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:39,155 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 2024-11-24T15:27:39,155 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a3f432e9db720f63dbb9d9a104c33f7 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-24T15:27:39,172 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/.tmp/a/a198ab51b2674a329938a91f64345ea8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1732462058737/Put/seqid=0 2024-11-24T15:27:39,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741852_1028 (size=5170) 2024-11-24T15:27:39,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741852_1028 (size=5170) 2024-11-24T15:27:39,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741852_1028 (size=5170) 2024-11-24T15:27:39,184 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/.tmp/a/a198ab51b2674a329938a91f64345ea8 2024-11-24T15:27:39,193 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/.tmp/a/a198ab51b2674a329938a91f64345ea8 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/a/a198ab51b2674a329938a91f64345ea8 2024-11-24T15:27:39,201 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/a/a198ab51b2674a329938a91f64345ea8, entries=2, sequenceid=2, filesize=5.0 K 2024-11-24T15:27:39,202 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8a3f432e9db720f63dbb9d9a104c33f7 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-24T15:27:39,202 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/0000000000000000002 2024-11-24T15:27:39,203 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,204 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,207 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a3f432e9db720f63dbb9d9a104c33f7 2024-11-24T15:27:39,211 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/8a3f432e9db720f63dbb9d9a104c33f7/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-24T15:27:39,212 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a3f432e9db720f63dbb9d9a104c33f7; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75265534, jitterRate=0.12154385447502136}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:27:39,213 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a3f432e9db720f63dbb9d9a104c33f7: Writing region info on filesystem at 1732462059113Initializing all the Stores at 1732462059115 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462059115Obtaining lock to block concurrent updates at 1732462059155 (+40 ms)Preparing flush snapshotting stores in 8a3f432e9db720f63dbb9d9a104c33f7 at 1732462059155Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1732462059155Flushing stores of testReplayEditsWrittenIntoWAL,,1732462058647.8a3f432e9db720f63dbb9d9a104c33f7. at 1732462059155Flushing 8a3f432e9db720f63dbb9d9a104c33f7/a: creating writer at 1732462059155Flushing 8a3f432e9db720f63dbb9d9a104c33f7/a: appending metadata at 1732462059171 (+16 ms)Flushing 8a3f432e9db720f63dbb9d9a104c33f7/a: closing flushed file at 1732462059171Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77f79d6a: reopening flushed file at 1732462059191 (+20 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 8a3f432e9db720f63dbb9d9a104c33f7 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1732462059202 (+11 ms)Cleaning up temporary data from old regions at 1732462059204 (+2 ms)Region opened successfully at 1732462059212 (+8 ms) 2024-11-24T15:27:39,232 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=385 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57582 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:36226 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:55562 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57782 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:36012 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:55540 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=251 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=6803 (was 6815) 2024-11-24T15:27:39,243 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=385, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=251, ProcessCount=11, AvailableMemoryMB=6803 2024-11-24T15:27:39,259 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:39,261 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:39,262 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:39,265 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54201118, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-54201118, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:39,280 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54201118/hregion-54201118.1732462059266, exclude list is [], retry=0 2024-11-24T15:27:39,285 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:39,286 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:39,286 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:39,317 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54201118/hregion-54201118.1732462059266 2024-11-24T15:27:39,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:39,318 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 39d9429663054a1748a3215ae1b14b6f, NAME => 'testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:39,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741854_1030 (size=64) 2024-11-24T15:27:39,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741854_1030 (size=64) 2024-11-24T15:27:39,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741854_1030 (size=64) 2024-11-24T15:27:39,331 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:39,333 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,335 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName a 2024-11-24T15:27:39,335 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:39,336 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:39,336 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,338 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName b 2024-11-24T15:27:39,338 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:39,338 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:39,338 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,340 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName c 2024-11-24T15:27:39,340 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:39,341 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:39,341 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,342 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,344 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,344 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,345 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:39,346 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:39,349 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:39,349 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39d9429663054a1748a3215ae1b14b6f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69106925, jitterRate=0.029773429036140442}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39d9429663054a1748a3215ae1b14b6f: Writing region info on filesystem at 1732462059331Initializing all the Stores at 1732462059333 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462059333Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462059333Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462059333Cleaning up temporary data from old regions at 1732462059344 (+11 ms)Region opened successfully at 1732462059349 (+5 ms) 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39d9429663054a1748a3215ae1b14b6f, disabling compactions & flushes 2024-11-24T15:27:39,350 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. after waiting 0 ms 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:39,350 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:39,350 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39d9429663054a1748a3215ae1b14b6f: Waiting for close lock at 1732462059350Disabling compacts and flushes for region at 1732462059350Disabling writes for close at 1732462059350Writing region close event to WAL at 1732462059350Closed at 1732462059350 2024-11-24T15:27:39,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741853_1029 (size=175) 2024-11-24T15:27:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741853_1029 (size=175) 2024-11-24T15:27:39,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741853_1029 (size=175) 2024-11-24T15:27:39,357 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:39,357 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54201118:(num 1732462059266) 2024-11-24T15:27:39,358 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:39,360 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:39,377 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, exclude list is [], retry=0 2024-11-24T15:27:39,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:39,382 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:39,382 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:39,422 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 2024-11-24T15:27:39,422 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:39,632 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, size=0 (0bytes) 2024-11-24T15:27:39,632 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 might be still open, length is 0 2024-11-24T15:27:39,632 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 2024-11-24T15:27:39,633 WARN [IPC Server handler 3 on default port 44579 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-24T15:27:39,633 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 after 1ms 2024-11-24T15:27:42,612 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:36256 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36256 dst: /127.0.0.1:41867 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41867 remote=/127.0.0.1:36256]. Total timeout mills is 60000, 56985 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:42,613 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:55610 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:55610 dst: /127.0.0.1:34483 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:42,613 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57812 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:39259:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57812 dst: /127.0.0.1:39259 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:42,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741855_1032 (size=497775) 2024-11-24T15:27:42,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741855_1032 (size=497775) 2024-11-24T15:27:42,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741855_1032 (size=497775) 2024-11-24T15:27:43,022 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T15:27:43,078 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-24T15:27:43,634 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 after 4002ms 2024-11-24T15:27:43,666 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:43,667 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 took 4035ms 2024-11-24T15:27:43,672 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732462059361.temp 2024-11-24T15:27:43,714 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp 2024-11-24T15:27:43,767 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361; continuing. 2024-11-24T15:27:43,767 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 so closing down 2024-11-24T15:27:43,768 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:43,768 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:43,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741856_1033 (size=497783) 2024-11-24T15:27:43,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741856_1033 (size=497783) 2024-11-24T15:27:43,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741856_1033 (size=497783) 2024-11-24T15:27:43,773 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp (wrote 3002 edits, skipped 0 edits in 41 ms) 2024-11-24T15:27:43,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 2024-11-24T15:27:43,776 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 108 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T15:27:43,776 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, journal: Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, size=0 (0bytes) at 1732462059632Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp at 1732462063714 (+4082 ms)Split 1024 edits, skipped 0 edits. at 1732462063723 (+9 ms)Split 2048 edits, skipped 0 edits. at 1732462063745 (+22 ms)Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 so closing down at 1732462063767 (+22 ms)3 split writer threads finished at 1732462063768 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp (wrote 3002 edits, skipped 0 edits in 41 ms) at 1732462063773 (+5 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000000001-wal.1732462059361.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 at 1732462063775 (+2 ms)Processed 3002 edits across 1 Regions in 108 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361, size=0, length=0, corrupted=false, cancelled=false at 1732462063776 (+1 ms) 2024-11-24T15:27:43,778 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462059361 2024-11-24T15:27:43,779 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 2024-11-24T15:27:43,780 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:43,782 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:43,796 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462063782, exclude list is [], retry=0 2024-11-24T15:27:43,799 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:43,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:43,800 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:43,829 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462063782 2024-11-24T15:27:43,829 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:43,829 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:43,831 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,832 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName a 2024-11-24T15:27:43,832 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:43,833 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:43,833 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,834 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName b 2024-11-24T15:27:43,835 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:43,835 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:43,835 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,836 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 39d9429663054a1748a3215ae1b14b6f columnFamilyName c 2024-11-24T15:27:43,836 DEBUG [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:43,837 INFO [StoreOpener-39d9429663054a1748a3215ae1b14b6f-1 {}] regionserver.HStore(327): Store=39d9429663054a1748a3215ae1b14b6f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:43,837 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,839 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,841 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:43,842 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 2024-11-24T15:27:43,873 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:43,918 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T15:27:44,318 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39d9429663054a1748a3215ae1b14b6f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T15:27:44,360 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/8630f48374b54e2bb1d103c73fb341a4 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1732462059429/Put/seqid=0 2024-11-24T15:27:44,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741858_1035 (size=50463) 2024-11-24T15:27:44,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741858_1035 (size=50463) 2024-11-24T15:27:44,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741858_1035 (size=50463) 2024-11-24T15:27:44,371 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/8630f48374b54e2bb1d103c73fb341a4 2024-11-24T15:27:44,380 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/8630f48374b54e2bb1d103c73fb341a4 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/a/8630f48374b54e2bb1d103c73fb341a4 2024-11-24T15:27:44,387 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/a/8630f48374b54e2bb1d103c73fb341a4, entries=754, sequenceid=754, filesize=49.3 K 2024-11-24T15:27:44,387 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 39d9429663054a1748a3215ae1b14b6f in 70ms, sequenceid=754, compaction requested=false; wal=null 2024-11-24T15:27:44,402 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T15:27:44,402 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39d9429663054a1748a3215ae1b14b6f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T15:27:44,409 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/444dc0eefeb6472d99d40589cb0388c3 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1732462059467/Put/seqid=0 2024-11-24T15:27:44,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741859_1036 (size=20072) 2024-11-24T15:27:44,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741859_1036 (size=20072) 2024-11-24T15:27:44,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741859_1036 (size=20072) 2024-11-24T15:27:44,418 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/444dc0eefeb6472d99d40589cb0388c3 2024-11-24T15:27:44,443 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/04bc74a804e64f779ac591c8041b5f01 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1732462059502/Put/seqid=0 2024-11-24T15:27:44,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741860_1037 (size=35835) 2024-11-24T15:27:44,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741860_1037 (size=35835) 2024-11-24T15:27:44,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741860_1037 (size=35835) 2024-11-24T15:27:44,459 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/04bc74a804e64f779ac591c8041b5f01 2024-11-24T15:27:44,467 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/a/444dc0eefeb6472d99d40589cb0388c3 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/a/444dc0eefeb6472d99d40589cb0388c3 2024-11-24T15:27:44,474 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/a/444dc0eefeb6472d99d40589cb0388c3, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-24T15:27:44,475 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/04bc74a804e64f779ac591c8041b5f01 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/b/04bc74a804e64f779ac591c8041b5f01 2024-11-24T15:27:44,482 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/b/04bc74a804e64f779ac591c8041b5f01, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-24T15:27:44,483 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 39d9429663054a1748a3215ae1b14b6f in 80ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-24T15:27:44,496 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T15:27:44,496 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39d9429663054a1748a3215ae1b14b6f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-24T15:27:44,504 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/5eacba779602427f9bfd913ea4e00a02 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1732462059521/Put/seqid=0 2024-11-24T15:27:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741861_1038 (size=35082) 2024-11-24T15:27:44,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741861_1038 (size=35082) 2024-11-24T15:27:44,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741861_1038 (size=35082) 2024-11-24T15:27:44,513 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/5eacba779602427f9bfd913ea4e00a02 2024-11-24T15:27:44,539 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/f600c1eac11540bea9c5b35b7048a032 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1732462059554/Put/seqid=0 2024-11-24T15:27:44,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741862_1039 (size=20825) 2024-11-24T15:27:44,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741862_1039 (size=20825) 2024-11-24T15:27:44,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741862_1039 (size=20825) 2024-11-24T15:27:44,548 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/f600c1eac11540bea9c5b35b7048a032 2024-11-24T15:27:44,556 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/b/5eacba779602427f9bfd913ea4e00a02 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/b/5eacba779602427f9bfd913ea4e00a02 2024-11-24T15:27:44,562 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/b/5eacba779602427f9bfd913ea4e00a02, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-24T15:27:44,563 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/f600c1eac11540bea9c5b35b7048a032 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/c/f600c1eac11540bea9c5b35b7048a032 2024-11-24T15:27:44,569 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/c/f600c1eac11540bea9c5b35b7048a032, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-24T15:27:44,569 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 39d9429663054a1748a3215ae1b14b6f in 73ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-24T15:27:44,579 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1732462059596/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:44,582 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 2024-11-24T15:27:44,583 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-24T15:27:44,583 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 39d9429663054a1748a3215ae1b14b6f 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-24T15:27:44,592 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/8c9d171e9145482e880aca6441de6955 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1732462059560/Put/seqid=0 2024-11-24T15:27:44,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741863_1040 (size=50301) 2024-11-24T15:27:44,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741863_1040 (size=50301) 2024-11-24T15:27:44,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741863_1040 (size=50301) 2024-11-24T15:27:44,602 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/8c9d171e9145482e880aca6441de6955 2024-11-24T15:27:44,608 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c9d171e9145482e880aca6441de6955 2024-11-24T15:27:44,610 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/.tmp/c/8c9d171e9145482e880aca6441de6955 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/c/8c9d171e9145482e880aca6441de6955 2024-11-24T15:27:44,616 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 8c9d171e9145482e880aca6441de6955 2024-11-24T15:27:44,616 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/c/8c9d171e9145482e880aca6441de6955, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-24T15:27:44,616 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 39d9429663054a1748a3215ae1b14b6f in 33ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-24T15:27:44,617 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/0000000000000003002 2024-11-24T15:27:44,619 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:44,619 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:44,619 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-24T15:27:44,621 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 39d9429663054a1748a3215ae1b14b6f 2024-11-24T15:27:44,624 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenIntoWAL/39d9429663054a1748a3215ae1b14b6f/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-24T15:27:44,625 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 39d9429663054a1748a3215ae1b14b6f; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73932176, jitterRate=0.10167527198791504}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-24T15:27:44,625 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 39d9429663054a1748a3215ae1b14b6f: Writing region info on filesystem at 1732462063830Initializing all the Stores at 1732462063831 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462063831Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462063831Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462063831Cleaning up temporary data from old regions at 1732462064619 (+788 ms)Region opened successfully at 1732462064625 (+6 ms) 2024-11-24T15:27:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 39d9429663054a1748a3215ae1b14b6f, disabling compactions & flushes 2024-11-24T15:27:44,683 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. after waiting 0 ms 2024-11-24T15:27:44,683 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:44,685 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1732462059260.39d9429663054a1748a3215ae1b14b6f. 2024-11-24T15:27:44,685 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 39d9429663054a1748a3215ae1b14b6f: Waiting for close lock at 1732462064682Disabling compacts and flushes for region at 1732462064682Disabling writes for close at 1732462064683 (+1 ms)Writing region close event to WAL at 1732462064685 (+2 ms)Closed at 1732462064685 2024-11-24T15:27:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741857_1034 (size=175) 2024-11-24T15:27:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741857_1034 (size=175) 2024-11-24T15:27:44,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741857_1034 (size=175) 2024-11-24T15:27:44,694 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:44,694 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732462063782) 2024-11-24T15:27:44,717 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 385) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1735028280_22 at /127.0.0.1:43708 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1735028280_22 at /127.0.0.1:43728 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38219 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1735028280_22 at /127.0.0.1:37362 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1735028280_22 at /127.0.0.1:54110 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:40197 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38219 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44579 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:40197 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 251), ProcessCount=11 (was 11), AvailableMemoryMB=6659 (was 6803) 2024-11-24T15:27:44,731 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=6658 2024-11-24T15:27:44,749 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:44,752 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:44,753 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:44,756 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45511956, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-45511956, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:44,771 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45511956/hregion-45511956.1732462064757, exclude list is [], retry=0 2024-11-24T15:27:44,775 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:44,775 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:44,775 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:44,820 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45511956/hregion-45511956.1732462064757 2024-11-24T15:27:44,820 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:44,820 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f13d705ffef00505fb37d7386b5549cf, NAME => 'test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741865_1042 (size=43) 2024-11-24T15:27:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741865_1042 (size=43) 2024-11-24T15:27:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741865_1042 (size=43) 2024-11-24T15:27:44,836 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:44,838 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,840 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName a 2024-11-24T15:27:44,840 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:44,840 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:44,841 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,844 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName b 2024-11-24T15:27:44,844 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:44,844 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:44,845 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,846 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName c 2024-11-24T15:27:44,846 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:44,847 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:44,847 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,847 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,848 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,849 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,849 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,850 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:44,851 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:44,854 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:44,854 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f13d705ffef00505fb37d7386b5549cf; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74065401, jitterRate=0.1036604791879654}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:44,855 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f13d705ffef00505fb37d7386b5549cf: Writing region info on filesystem at 1732462064836Initializing all the Stores at 1732462064838 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462064838Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462064838Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462064838Cleaning up temporary data from old regions at 1732462064849 (+11 ms)Region opened successfully at 1732462064855 (+6 ms) 2024-11-24T15:27:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f13d705ffef00505fb37d7386b5549cf, disabling compactions & flushes 2024-11-24T15:27:44,856 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. after waiting 0 ms 2024-11-24T15:27:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:44,856 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f13d705ffef00505fb37d7386b5549cf: Waiting for close lock at 1732462064855Disabling compacts and flushes for region at 1732462064855Disabling writes for close at 1732462064856 (+1 ms)Writing region close event to WAL at 1732462064856Closed at 1732462064856 2024-11-24T15:27:44,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741864_1041 (size=175) 2024-11-24T15:27:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741864_1041 (size=175) 2024-11-24T15:27:44,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741864_1041 (size=175) 2024-11-24T15:27:44,863 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:44,863 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45511956:(num 1732462064757) 2024-11-24T15:27:44,863 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:44,865 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:44,880 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, exclude list is [], retry=0 2024-11-24T15:27:44,883 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:44,883 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:44,883 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:44,924 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 2024-11-24T15:27:44,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741866_1043 (size=371388) 2024-11-24T15:27:45,085 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 not finished, retry = 0 2024-11-24T15:27:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741866_1043 (size=371388) 2024-11-24T15:27:45,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741866_1043 (size=371388) 2024-11-24T15:27:45,205 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, size=362.7 K (371388bytes) 2024-11-24T15:27:45,205 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 2024-11-24T15:27:45,206 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 after 0ms 2024-11-24T15:27:45,240 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:45,241 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 took 37ms 2024-11-24T15:27:45,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T15:27:45,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:45,246 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1732462064866.temp 2024-11-24T15:27:45,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T15:27:45,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-24T15:27:45,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-24T15:27:45,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:45,248 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-24T15:27:45,248 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:45,275 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp 2024-11-24T15:27:45,303 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 so closing down 2024-11-24T15:27:45,303 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:45,303 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:45,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741867_1044 (size=371388) 2024-11-24T15:27:45,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741867_1044 (size=371388) 2024-11-24T15:27:45,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741867_1044 (size=371388) 2024-11-24T15:27:45,319 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp (wrote 3000 edits, skipped 0 edits in 22 ms) 2024-11-24T15:27:45,321 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp to hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 2024-11-24T15:27:45,321 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 79 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-11-24T15:27:45,321 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, journal: Splitting hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, size=362.7 K (371388bytes) at 1732462065205Split 1024 edits, skipped 0 edits. at 1732462065265 (+60 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp at 1732462065275 (+10 ms)Split 2048 edits, skipped 0 edits. at 1732462065287 (+12 ms)Finishing writing output for hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 so closing down at 1732462065303 (+16 ms)3 split writer threads finished at 1732462065303Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp (wrote 3000 edits, skipped 0 edits in 22 ms) at 1732462065319 (+16 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000000001-wal.1732462064866.temp to hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 at 1732462065321 (+2 ms)Processed 3000 edits across 1 Regions in 79 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1732462065321 2024-11-24T15:27:45,324 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462064866 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462064866 2024-11-24T15:27:45,325 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 2024-11-24T15:27:45,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:45,327 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:45,346 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, exclude list is [], retry=0 2024-11-24T15:27:45,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:45,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:45,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:45,381 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 2024-11-24T15:27:45,382 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:27:45,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741868_1045 (size=371515) 2024-11-24T15:27:45,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741868_1045 (size=371515) 2024-11-24T15:27:45,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741868_1045 (size=371515) 2024-11-24T15:27:45,585 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, size=362.8 K (371515bytes) 2024-11-24T15:27:45,585 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 2024-11-24T15:27:45,585 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 after 0ms 2024-11-24T15:27:45,613 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:45,615 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 took 31ms 2024-11-24T15:27:45,619 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1732462065328.temp 2024-11-24T15:27:45,640 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 so closing down 2024-11-24T15:27:45,640 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:45,650 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp 2024-11-24T15:27:45,661 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:45,663 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741869_1046 (size=371515) 2024-11-24T15:27:45,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741869_1046 (size=371515) 2024-11-24T15:27:45,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741869_1046 (size=371515) 2024-11-24T15:27:45,665 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp (wrote 3000 edits, skipped 0 edits in 10 ms) 2024-11-24T15:27:45,666 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp to hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 2024-11-24T15:27:45,666 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-11-24T15:27:45,667 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, journal: Splitting hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, size=362.8 K (371515bytes) at 1732462065585Split 1024 edits, skipped 0 edits. at 1732462065625 (+40 ms)Split 2048 edits, skipped 0 edits. at 1732462065634 (+9 ms)Finishing writing output for hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 so closing down at 1732462065640 (+6 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp at 1732462065650 (+10 ms)3 split writer threads finished at 1732462065661 (+11 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp (wrote 3000 edits, skipped 0 edits in 10 ms) at 1732462065665 (+4 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003001-wal.1732462065328.temp to hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 at 1732462065666 (+1 ms)Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1732462065666 2024-11-24T15:27:45,668 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065328 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462065328 2024-11-24T15:27:45,669 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 2024-11-24T15:27:45,669 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:45,671 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/test2727-manual,16010,1732462064748, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:45,685 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065672, exclude list is [], retry=0 2024-11-24T15:27:45,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:45,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:45,688 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:45,716 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1732462064748/wal.1732462065672 2024-11-24T15:27:45,716 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:45,717 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f13d705ffef00505fb37d7386b5549cf, NAME => 'test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:45,717 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:45,717 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,717 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,719 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,720 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName a 2024-11-24T15:27:45,720 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:45,720 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:45,720 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,721 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName b 2024-11-24T15:27:45,721 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:45,722 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:45,722 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,723 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f13d705ffef00505fb37d7386b5549cf columnFamilyName c 2024-11-24T15:27:45,723 DEBUG [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:45,723 INFO [StoreOpener-f13d705ffef00505fb37d7386b5549cf-1 {}] regionserver.HStore(327): Store=f13d705ffef00505fb37d7386b5549cf/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:45,724 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,724 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,727 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:45,727 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 2024-11-24T15:27:45,771 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:45,801 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 2024-11-24T15:27:45,802 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 2024-11-24T15:27:45,831 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:45,855 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 2024-11-24T15:27:45,855 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f13d705ffef00505fb37d7386b5549cf 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-24T15:27:45,878 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/a/9652365aff9d4c4db9fd23a06bd475c6 is 41, key is test2727/a:100/1732462065387/Put/seqid=0 2024-11-24T15:27:45,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741871_1048 (size=84227) 2024-11-24T15:27:45,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741871_1048 (size=84227) 2024-11-24T15:27:45,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741871_1048 (size=84227) 2024-11-24T15:27:45,896 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/a/9652365aff9d4c4db9fd23a06bd475c6 2024-11-24T15:27:45,947 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/b/803f44f9bb314bce8c7f3afa1da2d3d7 is 41, key is test2727/b:100/1732462065438/Put/seqid=0 2024-11-24T15:27:45,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741872_1049 (size=84609) 2024-11-24T15:27:45,981 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741872_1049 (size=84609) 2024-11-24T15:27:45,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741872_1049 (size=84609) 2024-11-24T15:27:45,985 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/b/803f44f9bb314bce8c7f3afa1da2d3d7 2024-11-24T15:27:46,025 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/c/39215a6b960743939a4f5b538ea55c60 is 41, key is test2727/c:100/1732462065486/Put/seqid=0 2024-11-24T15:27:46,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741873_1050 (size=84609) 2024-11-24T15:27:46,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741873_1050 (size=84609) 2024-11-24T15:27:46,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741873_1050 (size=84609) 2024-11-24T15:27:46,043 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/c/39215a6b960743939a4f5b538ea55c60 2024-11-24T15:27:46,053 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/a/9652365aff9d4c4db9fd23a06bd475c6 as hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/a/9652365aff9d4c4db9fd23a06bd475c6 2024-11-24T15:27:46,062 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/a/9652365aff9d4c4db9fd23a06bd475c6, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-24T15:27:46,063 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/b/803f44f9bb314bce8c7f3afa1da2d3d7 as hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/b/803f44f9bb314bce8c7f3afa1da2d3d7 2024-11-24T15:27:46,072 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/b/803f44f9bb314bce8c7f3afa1da2d3d7, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T15:27:46,074 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/.tmp/c/39215a6b960743939a4f5b538ea55c60 as hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/c/39215a6b960743939a4f5b538ea55c60 2024-11-24T15:27:46,084 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/c/39215a6b960743939a4f5b538ea55c60, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-24T15:27:46,084 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f13d705ffef00505fb37d7386b5549cf in 229ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-24T15:27:46,085 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000003000 2024-11-24T15:27:46,086 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/0000000000000006000 2024-11-24T15:27:46,087 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:46,087 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:46,088 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:46,090 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f13d705ffef00505fb37d7386b5549cf 2024-11-24T15:27:46,093 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/test2727/f13d705ffef00505fb37d7386b5549cf/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-24T15:27:46,094 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f13d705ffef00505fb37d7386b5549cf; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66827628, jitterRate=-0.0041907429695129395}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:46,095 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f13d705ffef00505fb37d7386b5549cf: Writing region info on filesystem at 1732462065717Initializing all the Stores at 1732462065718 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462065718Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462065718Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462065718Obtaining lock to block concurrent updates at 1732462065855 (+137 ms)Preparing flush snapshotting stores in f13d705ffef00505fb37d7386b5549cf at 1732462065855Finished memstore snapshotting test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1732462065855Flushing stores of test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. at 1732462065855Flushing f13d705ffef00505fb37d7386b5549cf/a: creating writer at 1732462065855Flushing f13d705ffef00505fb37d7386b5549cf/a: appending metadata at 1732462065877 (+22 ms)Flushing f13d705ffef00505fb37d7386b5549cf/a: closing flushed file at 1732462065877Flushing f13d705ffef00505fb37d7386b5549cf/b: creating writer at 1732462065908 (+31 ms)Flushing f13d705ffef00505fb37d7386b5549cf/b: appending metadata at 1732462065945 (+37 ms)Flushing f13d705ffef00505fb37d7386b5549cf/b: closing flushed file at 1732462065945Flushing f13d705ffef00505fb37d7386b5549cf/c: creating writer at 1732462065994 (+49 ms)Flushing f13d705ffef00505fb37d7386b5549cf/c: appending metadata at 1732462066024 (+30 ms)Flushing f13d705ffef00505fb37d7386b5549cf/c: closing flushed file at 1732462066024Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@55fbceb: reopening flushed file at 1732462066052 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd3f153: reopening flushed file at 1732462066062 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a31ee04: reopening flushed file at 1732462066072 (+10 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for f13d705ffef00505fb37d7386b5549cf in 229ms, sequenceid=6000, compaction requested=false; wal=null at 1732462066084 (+12 ms)Cleaning up temporary data from old regions at 1732462066087 (+3 ms)Region opened successfully at 1732462066095 (+8 ms) 2024-11-24T15:27:46,096 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-24T15:27:46,097 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f13d705ffef00505fb37d7386b5549cf, disabling compactions & flushes 2024-11-24T15:27:46,097 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:46,097 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:46,097 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. after waiting 0 ms 2024-11-24T15:27:46,097 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:46,099 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1732462064750.f13d705ffef00505fb37d7386b5549cf. 2024-11-24T15:27:46,099 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f13d705ffef00505fb37d7386b5549cf: Waiting for close lock at 1732462066096Disabling compacts and flushes for region at 1732462066096Disabling writes for close at 1732462066097 (+1 ms)Writing region close event to WAL at 1732462066099 (+2 ms)Closed at 1732462066099 2024-11-24T15:27:46,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741870_1047 (size=175) 2024-11-24T15:27:46,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741870_1047 (size=175) 2024-11-24T15:27:46,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741870_1047 (size=175) 2024-11-24T15:27:46,106 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:46,106 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732462065672) 2024-11-24T15:27:46,123 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=403 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:37460 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:54254 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43728 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=6447 (was 6658) 2024-11-24T15:27:46,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=403, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=6446 2024-11-24T15:27:46,157 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:46,164 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:46,165 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1732462066164 2024-11-24T15:27:46,204 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 2024-11-24T15:27:46,209 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:46,213 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2fd63d88d44096fddafed3e6596182b4, NAME => 'testSequentialEditLogSeqNum,,1732462066158.2fd63d88d44096fddafed3e6596182b4.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:46,214 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1732462066158.2fd63d88d44096fddafed3e6596182b4.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:46,214 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,214 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,215 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4 doesn't exist for region: 2fd63d88d44096fddafed3e6596182b4 on table testSequentialEditLogSeqNum 2024-11-24T15:27:46,216 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 2fd63d88d44096fddafed3e6596182b4 on table testSequentialEditLogSeqNum 2024-11-24T15:27:46,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741875_1052 (size=62) 2024-11-24T15:27:46,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741875_1052 (size=62) 2024-11-24T15:27:46,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741875_1052 (size=62) 2024-11-24T15:27:46,232 INFO [StoreOpener-2fd63d88d44096fddafed3e6596182b4-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,234 INFO [StoreOpener-2fd63d88d44096fddafed3e6596182b4-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2fd63d88d44096fddafed3e6596182b4 columnFamilyName a 2024-11-24T15:27:46,234 DEBUG [StoreOpener-2fd63d88d44096fddafed3e6596182b4-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,235 INFO [StoreOpener-2fd63d88d44096fddafed3e6596182b4-1 {}] regionserver.HStore(327): Store=2fd63d88d44096fddafed3e6596182b4/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,236 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,238 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2fd63d88d44096fddafed3e6596182b4 2024-11-24T15:27:46,241 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:46,241 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2fd63d88d44096fddafed3e6596182b4; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71274628, jitterRate=0.06207472085952759}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:27:46,242 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2fd63d88d44096fddafed3e6596182b4: Writing region info on filesystem at 1732462066214Initializing all the Stores at 1732462066232 (+18 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066232Cleaning up temporary data from old regions at 1732462066236 (+4 ms)Region opened successfully at 1732462066242 (+6 ms) 2024-11-24T15:27:46,257 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2fd63d88d44096fddafed3e6596182b4 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-24T15:27:46,283 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/.tmp/a/dbbfd19db32a40298ee7119483f6e52f is 81, key is testSequentialEditLogSeqNum/a:x0/1732462066242/Put/seqid=0 2024-11-24T15:27:46,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741876_1053 (size=5833) 2024-11-24T15:27:46,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741876_1053 (size=5833) 2024-11-24T15:27:46,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741876_1053 (size=5833) 2024-11-24T15:27:46,291 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/.tmp/a/dbbfd19db32a40298ee7119483f6e52f 2024-11-24T15:27:46,298 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/.tmp/a/dbbfd19db32a40298ee7119483f6e52f as hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/a/dbbfd19db32a40298ee7119483f6e52f 2024-11-24T15:27:46,304 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/a/dbbfd19db32a40298ee7119483f6e52f, entries=10, sequenceid=13, filesize=5.7 K 2024-11-24T15:27:46,306 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 2fd63d88d44096fddafed3e6596182b4 in 49ms, sequenceid=13, compaction requested=false 2024-11-24T15:27:46,306 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2fd63d88d44096fddafed3e6596182b4: 2024-11-24T15:27:46,313 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T15:27:46,313 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T15:27:46,313 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T15:27:46,313 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T15:27:46,313 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-24T15:27:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741874_1051 (size=3256) 2024-11-24T15:27:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741874_1051 (size=3256) 2024-11-24T15:27:46,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741874_1051 (size=3256) 2024-11-24T15:27:46,333 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164, size=3.2 K (3256bytes) 2024-11-24T15:27:46,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 2024-11-24T15:27:46,333 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 after 0ms 2024-11-24T15:27:46,370 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:46,370 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 took 37ms 2024-11-24T15:27:46,373 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 so closing down 2024-11-24T15:27:46,373 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:46,376 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732462066164.temp 2024-11-24T15:27:46,404 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp 2024-11-24T15:27:46,405 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741877_1054 (size=2650) 2024-11-24T15:27:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741877_1054 (size=2650) 2024-11-24T15:27:46,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741877_1054 (size=2650) 2024-11-24T15:27:46,421 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:46,423 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp to hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000020 2024-11-24T15:27:46,424 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-11-24T15:27:46,424 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164, journal: Splitting hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164, size=3.2 K (3256bytes) at 1732462066333Finishing writing output for hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164 so closing down at 1732462066373 (+40 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp at 1732462066404 (+31 ms)3 split writer threads finished at 1732462066405 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1732462066421 (+16 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000003-wal.1732462066164.temp to hdfs://localhost:44579/hbase/data/default/testSequentialEditLogSeqNum/2fd63d88d44096fddafed3e6596182b4/recovered.edits/0000000000000000020 at 1732462066423 (+2 ms)Processed 17 edits across 1 Regions in 52 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1732462066156/wal.1732462066164, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1732462066424 (+1 ms) 2024-11-24T15:27:46,441 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=408 (was 403) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:37460 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43728 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=231 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=6442 (was 6446) 2024-11-24T15:27:46,453 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=231, ProcessCount=11, AvailableMemoryMB=6442 2024-11-24T15:27:46,472 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:46,475 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:46,505 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:46,508 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-96792584, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-96792584, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:46,522 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-96792584/hregion-96792584.1732462066508, exclude list is [], retry=0 2024-11-24T15:27:46,526 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:46,526 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:46,526 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:46,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-96792584/hregion-96792584.1732462066508 2024-11-24T15:27:46,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:27:46,555 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3282ab98d84baaad2a403f87763c524b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:46,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741879_1056 (size=70) 2024-11-24T15:27:46,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741879_1056 (size=70) 2024-11-24T15:27:46,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741879_1056 (size=70) 2024-11-24T15:27:46,580 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:46,582 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,584 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName a 2024-11-24T15:27:46,584 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,584 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,585 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,586 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName b 2024-11-24T15:27:46,586 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,587 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,587 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,588 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName c 2024-11-24T15:27:46,588 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,589 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,589 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,590 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,591 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,591 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,592 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:46,593 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,595 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:46,596 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3282ab98d84baaad2a403f87763c524b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65096984, jitterRate=-0.029979348182678223}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:46,597 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3282ab98d84baaad2a403f87763c524b: Writing region info on filesystem at 1732462066580Initializing all the Stores at 1732462066582 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066582Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066582Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066582Cleaning up temporary data from old regions at 1732462066591 (+9 ms)Region opened successfully at 1732462066597 (+6 ms) 2024-11-24T15:27:46,597 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3282ab98d84baaad2a403f87763c524b, disabling compactions & flushes 2024-11-24T15:27:46,597 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:46,597 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:46,597 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. after waiting 0 ms 2024-11-24T15:27:46,597 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:46,598 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:46,598 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3282ab98d84baaad2a403f87763c524b: Waiting for close lock at 1732462066597Disabling compacts and flushes for region at 1732462066597Disabling writes for close at 1732462066597Writing region close event to WAL at 1732462066598 (+1 ms)Closed at 1732462066598 2024-11-24T15:27:46,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741878_1055 (size=175) 2024-11-24T15:27:46,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741878_1055 (size=175) 2024-11-24T15:27:46,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741878_1055 (size=175) 2024-11-24T15:27:46,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:46,605 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-96792584:(num 1732462066508) 2024-11-24T15:27:46,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:46,607 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:46,626 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, exclude list is [], retry=0 2024-11-24T15:27:46,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:46,630 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:46,631 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:46,662 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 2024-11-24T15:27:46,663 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:27:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3282ab98d84baaad2a403f87763c524b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,663 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,665 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,666 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName a 2024-11-24T15:27:46,666 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,667 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,667 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,668 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName b 2024-11-24T15:27:46,668 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,669 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,669 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,670 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName c 2024-11-24T15:27:46,670 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:46,670 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:46,670 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,671 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,672 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,673 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,673 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,674 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:46,675 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:46,676 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3282ab98d84baaad2a403f87763c524b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72438093, jitterRate=0.07941170036792755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:46,677 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3282ab98d84baaad2a403f87763c524b: Writing region info on filesystem at 1732462066663Initializing all the Stores at 1732462066664 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066664Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066665 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462066665Cleaning up temporary data from old regions at 1732462066673 (+8 ms)Region opened successfully at 1732462066677 (+4 ms) 2024-11-24T15:27:46,681 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1732462066680/Put/seqid=0 2024-11-24T15:27:46,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741881_1058 (size=4826) 2024-11-24T15:27:46,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741881_1058 (size=4826) 2024-11-24T15:27:46,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741881_1058 (size=4826) 2024-11-24T15:27:46,689 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44579/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 3282ab98d84baaad2a403f87763c524b/a 2024-11-24T15:27:46,697 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-24T15:27:46,697 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T15:27:46,698 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3282ab98d84baaad2a403f87763c524b: 2024-11-24T15:27:46,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_ 2024-11-24T15:27:46,700 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44579/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3282ab98d84baaad2a403f87763c524b/a as hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_ - updating store file list. 2024-11-24T15:27:46,705 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T15:27:46,705 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_ into 3282ab98d84baaad2a403f87763c524b/a 2024-11-24T15:27:46,706 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44579/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 3282ab98d84baaad2a403f87763c524b/a (new location: hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_) 2024-11-24T15:27:46,756 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, size=0 (0bytes) 2024-11-24T15:27:46,757 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 might be still open, length is 0 2024-11-24T15:27:46,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 2024-11-24T15:27:46,757 WARN [IPC Server handler 2 on default port 44579 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-24T15:27:46,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 after 0ms 2024-11-24T15:27:48,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43970 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43970 dst: /127.0.0.1:34483 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34483 remote=/127.0.0.1:43970]. Total timeout mills is 60000, 57953 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:48,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:37552 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37552 dst: /127.0.0.1:41867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:48,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:54340 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:39259:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:54340 dst: /127.0.0.1:39259 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:48,764 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741880_1059 (size=606) 2024-11-24T15:27:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741880_1059 (size=606) 2024-11-24T15:27:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741880_1059 (size=606) 2024-11-24T15:27:50,758 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 after 4001ms 2024-11-24T15:27:50,787 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:50,787 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 took 4031ms 2024-11-24T15:27:50,789 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607; continuing. 2024-11-24T15:27:50,789 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 so closing down 2024-11-24T15:27:50,789 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:50,791 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1732462066607.temp 2024-11-24T15:27:50,816 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp 2024-11-24T15:27:50,817 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:50,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741882_1060 (size=329) 2024-11-24T15:27:50,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741882_1060 (size=329) 2024-11-24T15:27:50,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741882_1060 (size=329) 2024-11-24T15:27:50,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:50,826 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp to hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 2024-11-24T15:27:50,827 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T15:27:50,827 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, journal: Splitting hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, size=0 (0bytes) at 1732462066757Finishing writing output for hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 so closing down at 1732462070789 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp at 1732462070816 (+27 ms)3 split writer threads finished at 1732462070817 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1732462070824 (+7 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005-wal.1732462066607.temp to hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 at 1732462070826 (+2 ms)Processed 2 edits across 1 Regions in 39 ms; skipped=1; WAL=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607, size=0, length=0, corrupted=false, cancelled=false at 1732462070827 (+1 ms) 2024-11-24T15:27:50,829 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462066607 2024-11-24T15:27:50,830 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 2024-11-24T15:27:50,830 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:50,832 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:50,846 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462070833, exclude list is [], retry=0 2024-11-24T15:27:50,849 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:50,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:50,850 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:50,878 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462070833 2024-11-24T15:27:50,878 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:27:50,878 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3282ab98d84baaad2a403f87763c524b, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:50,879 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:50,879 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,879 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,880 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,881 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName a 2024-11-24T15:27:50,881 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:50,888 DEBUG [StoreFileOpener-3282ab98d84baaad2a403f87763c524b-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-24T15:27:50,888 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/19b358f9e1c6487d839e7f5e9d164f8f_SeqId_3_ 2024-11-24T15:27:50,888 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:50,888 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,889 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName b 2024-11-24T15:27:50,889 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:50,890 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:50,890 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,891 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3282ab98d84baaad2a403f87763c524b columnFamilyName c 2024-11-24T15:27:50,891 DEBUG [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:50,891 INFO [StoreOpener-3282ab98d84baaad2a403f87763c524b-1 {}] regionserver.HStore(327): Store=3282ab98d84baaad2a403f87763c524b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:50,891 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,892 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,894 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,894 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 2024-11-24T15:27:50,922 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:50,923 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 2024-11-24T15:27:50,923 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3282ab98d84baaad2a403f87763c524b 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-24T15:27:50,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/.tmp/a/a2af3adc18434673b0f00692e49b9967 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1732462066711/Put/seqid=0 2024-11-24T15:27:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741884_1062 (size=5149) 2024-11-24T15:27:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741884_1062 (size=5149) 2024-11-24T15:27:50,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741884_1062 (size=5149) 2024-11-24T15:27:50,945 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/.tmp/a/a2af3adc18434673b0f00692e49b9967 2024-11-24T15:27:50,950 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/.tmp/a/a2af3adc18434673b0f00692e49b9967 as hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/a2af3adc18434673b0f00692e49b9967 2024-11-24T15:27:50,956 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/a/a2af3adc18434673b0f00692e49b9967, entries=1, sequenceid=5, filesize=5.0 K 2024-11-24T15:27:50,956 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3282ab98d84baaad2a403f87763c524b in 33ms, sequenceid=5, compaction requested=false; wal=null 2024-11-24T15:27:50,956 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/0000000000000000005 2024-11-24T15:27:50,958 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,958 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,958 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:50,960 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3282ab98d84baaad2a403f87763c524b 2024-11-24T15:27:50,962 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/3282ab98d84baaad2a403f87763c524b/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-24T15:27:50,963 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3282ab98d84baaad2a403f87763c524b; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65367472, jitterRate=-0.025948762893676758}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:50,963 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3282ab98d84baaad2a403f87763c524b: Writing region info on filesystem at 1732462070879Initializing all the Stores at 1732462070880 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462070880Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462070880Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462070880Obtaining lock to block concurrent updates at 1732462070923 (+43 ms)Preparing flush snapshotting stores in 3282ab98d84baaad2a403f87763c524b at 1732462070923Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1732462070923Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. at 1732462070923Flushing 3282ab98d84baaad2a403f87763c524b/a: creating writer at 1732462070924 (+1 ms)Flushing 3282ab98d84baaad2a403f87763c524b/a: appending metadata at 1732462070937 (+13 ms)Flushing 3282ab98d84baaad2a403f87763c524b/a: closing flushed file at 1732462070938 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6dfadda2: reopening flushed file at 1732462070950 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 3282ab98d84baaad2a403f87763c524b in 33ms, sequenceid=5, compaction requested=false; wal=null at 1732462070956 (+6 ms)Cleaning up temporary data from old regions at 1732462070958 (+2 ms)Region opened successfully at 1732462070963 (+5 ms) 2024-11-24T15:27:50,967 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3282ab98d84baaad2a403f87763c524b, disabling compactions & flushes 2024-11-24T15:27:50,967 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:50,967 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:50,967 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. after waiting 0 ms 2024-11-24T15:27:50,967 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:50,968 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1732462066473.3282ab98d84baaad2a403f87763c524b. 2024-11-24T15:27:50,968 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3282ab98d84baaad2a403f87763c524b: Waiting for close lock at 1732462070967Disabling compacts and flushes for region at 1732462070967Disabling writes for close at 1732462070967Writing region close event to WAL at 1732462070968 (+1 ms)Closed at 1732462070968 2024-11-24T15:27:50,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741883_1061 (size=175) 2024-11-24T15:27:50,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741883_1061 (size=175) 2024-11-24T15:27:50,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741883_1061 (size=175) 2024-11-24T15:27:50,973 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:50,973 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732462070833) 2024-11-24T15:27:50,987 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 408) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44579 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:42061 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:42061 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_320619214_22 at /127.0.0.1:60910 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_320619214_22 at /127.0.0.1:54344 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=228 (was 231), ProcessCount=11 (was 11), AvailableMemoryMB=6432 (was 6442) 2024-11-24T15:27:50,998 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=228, ProcessCount=11, AvailableMemoryMB=6432 2024-11-24T15:27:51,014 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:51,019 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-24T15:27:51,023 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c99348da3e93,40561,1732462054824 2024-11-24T15:27:51,026 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@1af70c74 2024-11-24T15:27:51,027 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-24T15:27:51,029 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55216, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-24T15:27:51,033 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-24T15:27:51,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T15:27:51,043 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-24T15:27:51,045 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-24T15:27:51,045 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:51,047 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-24T15:27:51,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T15:27:51,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741885_1063 (size=694) 2024-11-24T15:27:51,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741885_1063 (size=694) 2024-11-24T15:27:51,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741885_1063 (size=694) 2024-11-24T15:27:51,060 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f 2024-11-24T15:27:51,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741886_1064 (size=77) 2024-11-24T15:27:51,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741886_1064 (size=77) 2024-11-24T15:27:51,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741886_1064 (size=77) 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:51,069 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,069 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,069 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462071069Disabling compacts and flushes for region at 1732462071069Disabling writes for close at 1732462071069Writing region close event to WAL at 1732462071069Closed at 1732462071069 2024-11-24T15:27:51,072 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-24T15:27:51,076 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1732462071072"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732462071072"}]},"ts":"1732462071072"} 2024-11-24T15:27:51,080 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-24T15:27:51,081 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-24T15:27:51,084 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732462071081"}]},"ts":"1732462071081"} 2024-11-24T15:27:51,088 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-24T15:27:51,088 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c99348da3e93=0} racks are {/default-rack=0} 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-24T15:27:51,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T15:27:51,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T15:27:51,089 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-24T15:27:51,089 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T15:27:51,091 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN}] 2024-11-24T15:27:51,092 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN 2024-11-24T15:27:51,094 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN; state=OFFLINE, location=c99348da3e93,41447,1732462055679; forceNewPlan=false, retain=false 2024-11-24T15:27:51,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T15:27:51,246 INFO [c99348da3e93:40561 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T15:27:51,247 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPENING, regionLocation=c99348da3e93,41447,1732462055679 2024-11-24T15:27:51,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN because future has completed 2024-11-24T15:27:51,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679}] 2024-11-24T15:27:51,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T15:27:51,404 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T15:27:51,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34813, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T15:27:51,411 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,411 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:51,411 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,412 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:51,412 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,412 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,413 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,414 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:51,414 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:51,415 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:51,415 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,416 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:51,416 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:51,417 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:51,417 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,418 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,418 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,419 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,419 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,420 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T15:27:51,421 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,423 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:51,424 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened bfa80e75ea1b7758b9c599d2ce0ef545; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60759485, jitterRate=-0.09461311995983124}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T15:27:51,424 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,424 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for bfa80e75ea1b7758b9c599d2ce0ef545: Running coprocessor pre-open hook at 1732462071412Writing region info on filesystem at 1732462071412Initializing all the Stores at 1732462071413 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462071413Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462071413Cleaning up temporary data from old regions at 1732462071419 (+6 ms)Running coprocessor post-open hooks at 1732462071424 (+5 ms)Region opened successfully at 1732462071424 2024-11-24T15:27:51,426 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., pid=6, masterSystemTime=1732462071404 2024-11-24T15:27:51,429 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,429 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,429 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPEN, openSeqNum=2, regionLocation=c99348da3e93,41447,1732462055679 2024-11-24T15:27:51,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 because future has completed 2024-11-24T15:27:51,437 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-24T15:27:51,438 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 in 184 msec 2024-11-24T15:27:51,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-24T15:27:51,441 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN in 347 msec 2024-11-24T15:27:51,442 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-24T15:27:51,442 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732462071442"}]},"ts":"1732462071442"} 2024-11-24T15:27:51,445 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-24T15:27:51,446 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-24T15:27:51,448 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 410 msec 2024-11-24T15:27:51,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-24T15:27:51,677 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-24T15:27:51,677 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-24T15:27:51,679 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T15:27:51,684 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-24T15:27:51,685 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-24T15:27:51,685 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-24T15:27:51,698 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=2] 2024-11-24T15:27:51,699 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T15:27:51,701 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T15:27:51,715 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=bfa80e75ea1b7758b9c599d2ce0ef545, source=c99348da3e93,41447,1732462055679, destination=c99348da3e93,38975,1732462055523, warming up region on c99348da3e93,38975,1732462055523 2024-11-24T15:27:51,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T15:27:51,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=bfa80e75ea1b7758b9c599d2ce0ef545, source=c99348da3e93,41447,1732462055679, destination=c99348da3e93,38975,1732462055523, running balancer 2024-11-24T15:27:51,718 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41111, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T15:27:51,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE 2024-11-24T15:27:51,719 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE 2024-11-24T15:27:51,721 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=CLOSING, regionLocation=c99348da3e93,41447,1732462055679 2024-11-24T15:27:51,722 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(7855): Warmup {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:51,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:51,723 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,724 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE because future has completed 2024-11-24T15:27:51,724 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T15:27:51,725 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:51,725 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:51,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679}] 2024-11-24T15:27:51,725 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:51,725 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,726 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:51,726 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:51,727 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:51,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:51,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462071727Disabling compacts and flushes for region at 1732462071727Disabling writes for close at 1732462071727Writing region close event to WAL at 1732462071728 (+1 ms)Closed at 1732462071728 2024-11-24T15:27:51,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-24T15:27:51,883 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,883 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T15:27:51,884 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:51,884 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,884 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,885 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:51,885 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,885 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing bfa80e75ea1b7758b9c599d2ce0ef545 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-24T15:27:51,903 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/19e1b3792ba44069b54276202e12e016 is 35, key is r1/cf1:q/1732462071702/Put/seqid=0 2024-11-24T15:27:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741887_1065 (size=4783) 2024-11-24T15:27:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741887_1065 (size=4783) 2024-11-24T15:27:51,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741887_1065 (size=4783) 2024-11-24T15:27:51,911 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/19e1b3792ba44069b54276202e12e016 2024-11-24T15:27:51,919 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/19e1b3792ba44069b54276202e12e016 as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016 2024-11-24T15:27:51,925 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016, entries=1, sequenceid=5, filesize=4.7 K 2024-11-24T15:27:51,927 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for bfa80e75ea1b7758b9c599d2ce0ef545 in 42ms, sequenceid=5, compaction requested=false 2024-11-24T15:27:51,927 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-24T15:27:51,932 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T15:27:51,935 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:51,935 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462071884Running coprocessor pre-close hooks at 1732462071884Disabling compacts and flushes for region at 1732462071884Disabling writes for close at 1732462071885 (+1 ms)Obtaining lock to block concurrent updates at 1732462071885Preparing flush snapshotting stores in bfa80e75ea1b7758b9c599d2ce0ef545 at 1732462071885Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1732462071885Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. at 1732462071886 (+1 ms)Flushing bfa80e75ea1b7758b9c599d2ce0ef545/cf1: creating writer at 1732462071886Flushing bfa80e75ea1b7758b9c599d2ce0ef545/cf1: appending metadata at 1732462071902 (+16 ms)Flushing bfa80e75ea1b7758b9c599d2ce0ef545/cf1: closing flushed file at 1732462071902Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@69c144f7: reopening flushed file at 1732462071918 (+16 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for bfa80e75ea1b7758b9c599d2ce0ef545 in 42ms, sequenceid=5, compaction requested=false at 1732462071927 (+9 ms)Writing region close event to WAL at 1732462071928 (+1 ms)Running coprocessor post-close hooks at 1732462071932 (+4 ms)Closed at 1732462071935 (+3 ms) 2024-11-24T15:27:51,935 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding bfa80e75ea1b7758b9c599d2ce0ef545 move to c99348da3e93,38975,1732462055523 record at close sequenceid=5 2024-11-24T15:27:51,938 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:51,939 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=CLOSED 2024-11-24T15:27:51,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 because future has completed 2024-11-24T15:27:51,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-24T15:27:51,948 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 in 219 msec 2024-11-24T15:27:51,949 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE; state=CLOSED, location=c99348da3e93,38975,1732462055523; forceNewPlan=false, retain=false 2024-11-24T15:27:52,100 INFO [c99348da3e93:40561 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T15:27:52,100 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPENING, regionLocation=c99348da3e93,38975,1732462055523 2024-11-24T15:27:52,103 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE because future has completed 2024-11-24T15:27:52,104 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523}] 2024-11-24T15:27:52,261 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,262 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:52,262 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,262 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:52,262 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,262 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,264 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,265 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:52,265 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:52,273 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016 2024-11-24T15:27:52,273 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:52,274 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,274 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:52,274 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:52,275 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:52,275 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,276 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,277 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,278 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,278 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,278 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T15:27:52,280 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,281 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened bfa80e75ea1b7758b9c599d2ce0ef545; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67598310, jitterRate=0.00729331374168396}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T15:27:52,281 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,282 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for bfa80e75ea1b7758b9c599d2ce0ef545: Running coprocessor pre-open hook at 1732462072262Writing region info on filesystem at 1732462072262Initializing all the Stores at 1732462072263 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462072263Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462072264 (+1 ms)Cleaning up temporary data from old regions at 1732462072278 (+14 ms)Running coprocessor post-open hooks at 1732462072281 (+3 ms)Region opened successfully at 1732462072282 (+1 ms) 2024-11-24T15:27:52,283 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., pid=9, masterSystemTime=1732462072257 2024-11-24T15:27:52,285 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,285 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,286 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPEN, openSeqNum=9, regionLocation=c99348da3e93,38975,1732462055523 2024-11-24T15:27:52,289 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 because future has completed 2024-11-24T15:27:52,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-24T15:27:52,293 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 in 186 msec 2024-11-24T15:27:52,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE in 576 msec 2024-11-24T15:27:52,325 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T15:27:52,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T15:27:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:41132 deadline: 1732462132330, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=38975 startCode=1732462055523. As of locationSeqNum=5. 2024-11-24T15:27:52,354 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=38975 startCode=1732462055523. As of locationSeqNum=5. 2024-11-24T15:27:52,355 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=38975 startCode=1732462055523. As of locationSeqNum=5. 2024-11-24T15:27:52,355 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=38975 startCode=1732462055523. As of locationSeqNum=5. 2024-11-24T15:27:52,466 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-24T15:27:52,467 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49184, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-24T15:27:52,479 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing bfa80e75ea1b7758b9c599d2ce0ef545 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-24T15:27:52,496 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/cedf5e4dc1154098b15a7507c62f078f is 29, key is r1/cf1:/1732462072469/DeleteFamily/seqid=0 2024-11-24T15:27:52,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741888_1066 (size=4906) 2024-11-24T15:27:52,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741888_1066 (size=4906) 2024-11-24T15:27:52,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741888_1066 (size=4906) 2024-11-24T15:27:52,505 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,513 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,536 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf2/bee408b754a14e2a9bfc997984a40277 is 29, key is r1/cf2:/1732462072469/DeleteFamily/seqid=0 2024-11-24T15:27:52,543 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741889_1067 (size=4906) 2024-11-24T15:27:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741889_1067 (size=4906) 2024-11-24T15:27:52,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741889_1067 (size=4906) 2024-11-24T15:27:52,544 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf2/bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,551 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,552 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/cedf5e4dc1154098b15a7507c62f078f as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,559 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,559 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T15:27:52,560 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf2/bee408b754a14e2a9bfc997984a40277 as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,566 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,567 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277, entries=1, sequenceid=12, filesize=4.8 K 2024-11-24T15:27:52,568 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for bfa80e75ea1b7758b9c599d2ce0ef545 in 89ms, sequenceid=12, compaction requested=false 2024-11-24T15:27:52,568 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for bfa80e75ea1b7758b9c599d2ce0ef545: 2024-11-24T15:27:52,570 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-24T15:27:52,572 DEBUG [Time-limited test {}] regionserver.HStore(1541): bfa80e75ea1b7758b9c599d2ce0ef545/cf1 is initiating major compaction (all files) 2024-11-24T15:27:52,572 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T15:27:52,572 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:52,572 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of bfa80e75ea1b7758b9c599d2ce0ef545/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,573 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016, hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f] into tmpdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp, totalSize=9.5 K 2024-11-24T15:27:52,574 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 19e1b3792ba44069b54276202e12e016, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1732462071702 2024-11-24T15:27:52,574 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting cedf5e4dc1154098b15a7507c62f078f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T15:27:52,585 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): bfa80e75ea1b7758b9c599d2ce0ef545#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T15:27:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741890_1068 (size=4626) 2024-11-24T15:27:52,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741890_1068 (size=4626) 2024-11-24T15:27:52,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741890_1068 (size=4626) 2024-11-24T15:27:52,601 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf1/ca58d88083af480795630317bee9c3da as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/ca58d88083af480795630317bee9c3da 2024-11-24T15:27:52,614 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in bfa80e75ea1b7758b9c599d2ce0ef545/cf1 of bfa80e75ea1b7758b9c599d2ce0ef545 into ca58d88083af480795630317bee9c3da(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T15:27:52,614 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for bfa80e75ea1b7758b9c599d2ce0ef545: 2024-11-24T15:27:52,615 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-24T15:27:52,615 DEBUG [Time-limited test {}] regionserver.HStore(1541): bfa80e75ea1b7758b9c599d2ce0ef545/cf2 is initiating major compaction (all files) 2024-11-24T15:27:52,615 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-24T15:27:52,615 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-24T15:27:52,615 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of bfa80e75ea1b7758b9c599d2ce0ef545/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,615 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277] into tmpdir=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp, totalSize=4.8 K 2024-11-24T15:27:52,616 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bee408b754a14e2a9bfc997984a40277, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-24T15:27:52,622 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): bfa80e75ea1b7758b9c599d2ce0ef545#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-24T15:27:52,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741891_1069 (size=4592) 2024-11-24T15:27:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741891_1069 (size=4592) 2024-11-24T15:27:52,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741891_1069 (size=4592) 2024-11-24T15:27:52,638 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/.tmp/cf2/5988c863654a4b1fbfe3f47705526a5b as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/5988c863654a4b1fbfe3f47705526a5b 2024-11-24T15:27:52,645 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in bfa80e75ea1b7758b9c599d2ce0ef545/cf2 of bfa80e75ea1b7758b9c599d2ce0ef545 into 5988c863654a4b1fbfe3f47705526a5b(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T15:27:52,645 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for bfa80e75ea1b7758b9c599d2ce0ef545: 2024-11-24T15:27:52,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=bfa80e75ea1b7758b9c599d2ce0ef545, source=c99348da3e93,38975,1732462055523, destination=c99348da3e93,41447,1732462055679, warming up region on c99348da3e93,41447,1732462055679 2024-11-24T15:27:52,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=bfa80e75ea1b7758b9c599d2ce0ef545, source=c99348da3e93,38975,1732462055523, destination=c99348da3e93,41447,1732462055679, running balancer 2024-11-24T15:27:52,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE 2024-11-24T15:27:52,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE 2024-11-24T15:27:52,654 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=CLOSING, regionLocation=c99348da3e93,38975,1732462055523 2024-11-24T15:27:52,655 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(7855): Warmup {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:52,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:52,656 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,657 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:52,657 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:52,662 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE because future has completed 2024-11-24T15:27:52,663 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-24T15:27:52,664 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523}] 2024-11-24T15:27:52,666 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016 2024-11-24T15:27:52,673 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/ca58d88083af480795630317bee9c3da 2024-11-24T15:27:52,679 INFO [StoreFileOpener-bfa80e75ea1b7758b9c599d2ce0ef545-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,679 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,679 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:52,679 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,681 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:52,681 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:52,688 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/5988c863654a4b1fbfe3f47705526a5b 2024-11-24T15:27:52,693 INFO [StoreFileOpener-bfa80e75ea1b7758b9c599d2ce0ef545-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,693 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,693 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:52,693 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:52,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41447 {}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462072693Disabling compacts and flushes for region at 1732462072693Disabling writes for close at 1732462072694 (+1 ms)Writing region close event to WAL at 1732462072695 (+1 ms)Closed at 1732462072695 2024-11-24T15:27:52,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-24T15:27:52,819 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,819 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-24T15:27:52,820 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:52,820 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,820 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,820 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:52,820 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,820 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016, hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f] to archive 2024-11-24T15:27:52,823 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T15:27:52,827 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016 to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/19e1b3792ba44069b54276202e12e016 2024-11-24T15:27:52,828 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/cedf5e4dc1154098b15a7507c62f078f 2024-11-24T15:27:52,841 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277] to archive 2024-11-24T15:27:52,842 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T15:27:52,844 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277 to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/bee408b754a14e2a9bfc997984a40277 2024-11-24T15:27:52,849 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-24T15:27:52,849 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:52,849 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462072820Running coprocessor pre-close hooks at 1732462072820Disabling compacts and flushes for region at 1732462072820Disabling writes for close at 1732462072820Writing region close event to WAL at 1732462072845 (+25 ms)Running coprocessor post-close hooks at 1732462072849 (+4 ms)Closed at 1732462072849 2024-11-24T15:27:52,849 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding bfa80e75ea1b7758b9c599d2ce0ef545 move to c99348da3e93,41447,1732462055679 record at close sequenceid=12 2024-11-24T15:27:52,853 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:52,854 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=CLOSED 2024-11-24T15:27:52,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 because future has completed 2024-11-24T15:27:52,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-24T15:27:52,861 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 in 194 msec 2024-11-24T15:27:52,862 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE; state=CLOSED, location=c99348da3e93,41447,1732462055679; forceNewPlan=false, retain=false 2024-11-24T15:27:53,013 INFO [c99348da3e93:40561 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-24T15:27:53,013 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPENING, regionLocation=c99348da3e93,41447,1732462055679 2024-11-24T15:27:53,016 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE because future has completed 2024-11-24T15:27:53,016 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679}] 2024-11-24T15:27:53,172 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,173 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:53,173 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,173 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:53,173 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,173 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,175 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,176 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:53,176 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:53,182 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/ca58d88083af480795630317bee9c3da 2024-11-24T15:27:53,182 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:53,182 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,183 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:53,183 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:53,189 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/5988c863654a4b1fbfe3f47705526a5b 2024-11-24T15:27:53,189 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:53,189 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,189 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,190 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,191 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,191 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,192 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T15:27:53,193 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,194 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened bfa80e75ea1b7758b9c599d2ce0ef545; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75163059, jitterRate=0.12001685798168182}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T15:27:53,194 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,195 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for bfa80e75ea1b7758b9c599d2ce0ef545: Running coprocessor pre-open hook at 1732462073173Writing region info on filesystem at 1732462073173Initializing all the Stores at 1732462073174 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462073174Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462073175 (+1 ms)Cleaning up temporary data from old regions at 1732462073191 (+16 ms)Running coprocessor post-open hooks at 1732462073194 (+3 ms)Region opened successfully at 1732462073195 (+1 ms) 2024-11-24T15:27:53,196 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., pid=12, masterSystemTime=1732462073168 2024-11-24T15:27:53,198 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,198 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,199 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPEN, openSeqNum=18, regionLocation=c99348da3e93,41447,1732462055679 2024-11-24T15:27:53,201 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 because future has completed 2024-11-24T15:27:53,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-24T15:27:53,205 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,41447,1732462055679 in 186 msec 2024-11-24T15:27:53,207 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, REOPEN/MOVE in 554 msec 2024-11-24T15:27:53,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-24T15:27:53,257 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-24T15:27:53,259 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c99348da3e93,41447,1732462055679: testing ***** 2024-11-24T15:27:53,259 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-24T15:27:53,261 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-24T15:27:53,262 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-24T15:27:53,266 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-24T15:27:53,267 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-24T15:27:53,277 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 513565440 }, "NonHeapMemoryUsage": { "committed": 172425216, "init": 7667712, "max": -1, "used": 169789656 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c99348da3e93", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11076, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 44, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 3, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 10, "ProcessCallTime_98th_percentile": 44, "ProcessCallTime_99th_percentile": 44, "ProcessCallTime_99.9th_percentile": 44, "ProcessCallTime_TimeRangeCount_0-1": 12, "ProcessCallTime_TimeRangeCount_1-3": 13, "ProcessCallTime_TimeRangeCount_3-10": 5, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 44, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 3, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 10, "TotalCallTime_98th_percentile": 44, "TotalCallTime_99th_percentile": 44, "TotalCallTime_99.9th_percentile": 44, "TotalCallTime_TimeRangeCount_0-1": 12, "TotalCallTime_TimeRangeCount_1-3": 13, "TotalCallTime_TimeRangeCount_3-10": 5, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1139, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 796, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1070, "ResponseSize_99.9th_percentile": 1132, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5752 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c99348da3e93", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:64743", "tag.serverName": "c99348da3e93,41447,1732462055679", "tag.clusterId": "357eacd2-e660-4f26-ba8c-7141b8360e78", "tag.Context": "regionserver", "tag.Hostname": "c99348da3e93", "regionCount": 1, "storeCount": 2, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 31, "memStoreHeapSize": 616, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1732462055679, "averageRegionSize": 31, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.2, "writeRequestRatePerSecond": 0.2, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 2, "readRequestCount": 1, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 1, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 3, "Put_max": 3, "Put_mean": 3, "Put_25th_percentile": 3, "Put_median": 3, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 42, "FlushTime_max": 42, "FlushTime_mean": 42, "FlushTime_25th_percentile": 42, "FlushTime_median": 42, "FlushTime_75th_percentile": 42, "FlushTime_90th_percentile": 42, "FlushTime_95th_percentile": 42, "FlushTime_98th_percentile": 42, "FlushTime_99th_percentile": 42, "FlushTime_99.9th_percentile": 42, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-24T15:27:53,280 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40561 {}] master.MasterRpcServices(700): c99348da3e93,41447,1732462055679 reported a fatal error: ***** ABORTING region server c99348da3e93,41447,1732462055679: testing ***** 2024-11-24T15:27:53,282 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c99348da3e93,41447,1732462055679' ***** 2024-11-24T15:27:53,282 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-24T15:27:53,282 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T15:27:53,283 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T15:27:53,283 INFO [RS:2;c99348da3e93:41447 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-24T15:27:53,283 INFO [RS:2;c99348da3e93:41447 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-24T15:27:53,283 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(3091): Received CLOSE for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,284 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(956): aborting server c99348da3e93,41447,1732462055679 2024-11-24T15:27:53,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38975 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:49184 deadline: 1732462133283, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=41447 startCode=1732462055679. As of locationSeqNum=12. 2024-11-24T15:27:53,284 INFO [RS:2;c99348da3e93:41447 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T15:27:53,284 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:27:53,284 INFO [RS:2;c99348da3e93:41447 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c99348da3e93:41447. 2024-11-24T15:27:53,284 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,284 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,284 DEBUG [RS:2;c99348da3e93:41447 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T15:27:53,284 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:27:53,284 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=41447 startCode=1732462055679. As of locationSeqNum=12. 2024-11-24T15:27:53,284 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,285 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=41447 startCode=1732462055679. As of locationSeqNum=12. 2024-11-24T15:27:53,285 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c99348da3e93 port=41447 startCode=1732462055679. As of locationSeqNum=12. 2024-11-24T15:27:53,285 DEBUG [RS:2;c99348da3e93:41447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:27:53,285 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T15:27:53,286 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1325): Online Regions={bfa80e75ea1b7758b9c599d2ce0ef545=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.} 2024-11-24T15:27:53,286 DEBUG [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1351): Waiting on bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:53,287 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,287 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462073284Running coprocessor pre-close hooks at 1732462073284Disabling compacts and flushes for region at 1732462073284Disabling writes for close at 1732462073284Writing region close event to WAL at 1732462073287 (+3 ms)Running coprocessor post-close hooks at 1732462073287Closed at 1732462073287 2024-11-24T15:27:53,287 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:53,352 INFO [regionserver/c99348da3e93:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:27:53,397 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c99348da3e93,41447,1732462055679 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:53,398 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c99348da3e93,41447,1732462055679 aborting 2024-11-24T15:27:53,398 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c99348da3e93,41447,1732462055679 aborting 2024-11-24T15:27:53,398 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=12 from cache 2024-11-24T15:27:53,486 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(976): stopping server c99348da3e93,41447,1732462055679; all regions closed. 2024-11-24T15:27:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741834_1010 (size=2288) 2024-11-24T15:27:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741834_1010 (size=2288) 2024-11-24T15:27:53,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741834_1010 (size=2288) 2024-11-24T15:27:53,491 DEBUG [RS:2;c99348da3e93:41447 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] hbase.ChoreService(370): Chore service for: regionserver/c99348da3e93:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T15:27:53,491 INFO [regionserver/c99348da3e93:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T15:27:53,491 INFO [RS:2;c99348da3e93:41447 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T15:27:53,492 INFO [RS:2;c99348da3e93:41447 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:41447 2024-11-24T15:27:53,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c99348da3e93,41447,1732462055679 2024-11-24T15:27:53,498 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T15:27:53,499 INFO [RS:2;c99348da3e93:41447 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T15:27:53,500 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c99348da3e93,41447,1732462055679] 2024-11-24T15:27:53,502 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c99348da3e93,41447,1732462055679 already deleted, retry=false 2024-11-24T15:27:53,502 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c99348da3e93,41447,1732462055679 on c99348da3e93,40561,1732462054824 2024-11-24T15:27:53,508 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c99348da3e93,41447,1732462055679, splitWal=true, meta=false 2024-11-24T15:27:53,510 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for c99348da3e93,41447,1732462055679 (carryingMeta=false) c99348da3e93,41447,1732462055679/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@3f3dd0e4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-24T15:27:53,511 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c99348da3e93,41447,1732462055679, splitWal=true, meta=false 2024-11-24T15:27:53,512 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): c99348da3e93,41447,1732462055679 had 1 regions 2024-11-24T15:27:53,513 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c99348da3e93,41447,1732462055679, splitWal=true, meta=false, isMeta: false 2024-11-24T15:27:53,515 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting 2024-11-24T15:27:53,516 INFO [PEWorker-2 {}] master.SplitWALManager(105): c99348da3e93,41447,1732462055679 WAL count=1, meta=false 2024-11-24T15:27:53,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c99348da3e93%2C41447%2C1732462055679.1732462056995}] 2024-11-24T15:27:53,523 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=c99348da3e93,33951,1732462055634 2024-11-24T15:27:53,525 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c99348da3e93%2C41447%2C1732462055679.1732462056995, worker=c99348da3e93,33951,1732462055634}] 2024-11-24T15:27:53,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:27:53,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41447-0x1014a96f51f0003, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:27:53,602 INFO [RS:2;c99348da3e93:41447 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T15:27:53,602 INFO [RS:2;c99348da3e93:41447 {}] regionserver.HRegionServer(1031): Exiting; stopping=c99348da3e93,41447,1732462055679; zookeeper connection closed. 2024-11-24T15:27:53,602 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@643f556c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@643f556c 2024-11-24T15:27:53,609 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18] 2024-11-24T15:27:53,611 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c99348da3e93:41447 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c99348da3e93/172.17.0.2:41447 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:53,611 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18, error=java.net.ConnectException: Call to address=c99348da3e93:41447 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c99348da3e93/172.17.0.2:41447 2024-11-24T15:27:53,612 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-24T15:27:53,612 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 from cache 2024-11-24T15:27:53,612 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address c99348da3e93:41447 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c99348da3e93/172.17.0.2:41447 2024-11-24T15:27:53,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=33951 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-24T15:27:53,705 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995, size=2.2 K (2288bytes) 2024-11-24T15:27:53,705 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 2024-11-24T15:27:53,705 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 after 0ms 2024-11-24T15:27:53,733 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:53,733 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 took 28ms 2024-11-24T15:27:53,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for bfa80e75ea1b7758b9c599d2ce0ef545: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-24T15:27:53,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 so closing down 2024-11-24T15:27:53,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:53,741 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:53,741 INFO [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-11-24T15:27:53,741 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995, journal: Splitting hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995, size=2.2 K (2288bytes) at 1732462073705Finishing writing output for hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 so closing down at 1732462073741 (+36 ms)3 split writer threads finished at 1732462073741Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1732462073741 2024-11-24T15:27:53,742 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 2024-11-24T15:27:53,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c99348da3e93:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-24T15:27:53,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40561 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-24T15:27:53,749 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting/c99348da3e93%2C41447%2C1732462055679.1732462056995 to hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs 2024-11-24T15:27:53,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-24T15:27:53,752 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c99348da3e93%2C41447%2C1732462055679.1732462056995, worker=c99348da3e93,33951,1732462055634 in 224 msec 2024-11-24T15:27:53,753 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=c99348da3e93,33951,1732462055634 2024-11-24T15:27:53,755 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-24T15:27:53,756 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c99348da3e93%2C41447%2C1732462055679.1732462056995, worker=c99348da3e93,33951,1732462055634 in 236 msec 2024-11-24T15:27:53,757 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting dir is empty, no logs to split. 2024-11-24T15:27:53,757 INFO [PEWorker-2 {}] master.SplitWALManager(105): c99348da3e93,41447,1732462055679 WAL count=0, meta=false 2024-11-24T15:27:53,757 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if c99348da3e93,41447,1732462055679 WAL splitting is done? wals=0, meta=false 2024-11-24T15:27:53,759 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c99348da3e93,41447,1732462055679 failed, ignore...File hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,41447,1732462055679-splitting does not exist. 2024-11-24T15:27:53,760 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN}] 2024-11-24T15:27:53,761 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN 2024-11-24T15:27:53,762 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(204): Hosts are {c99348da3e93=0} racks are {/default-rack=0} 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-24T15:27:53,913 INFO [c99348da3e93:40561 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-24T15:27:53,913 INFO [c99348da3e93:40561 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-24T15:27:53,913 DEBUG [c99348da3e93:40561 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-24T15:27:53,914 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPENING, regionLocation=c99348da3e93,38975,1732462055523 2024-11-24T15:27:53,917 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN because future has completed 2024-11-24T15:27:53,917 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523}] 2024-11-24T15:27:53,919 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18] 2024-11-24T15:27:53,919 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to c99348da3e93:41447 this server is in the failed servers list 2024-11-24T15:27:53,920 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c99348da3e93:41447 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c99348da3e93:41447 2024-11-24T15:27:53,920 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c99348da3e93:41447 2024-11-24T15:27:53,921 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,41447,1732462055679, seqNum=18 from cache 2024-11-24T15:27:54,074 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:54,074 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => bfa80e75ea1b7758b9c599d2ce0ef545, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:54,075 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,075 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:54,075 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,075 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,076 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,077 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf1 2024-11-24T15:27:54,077 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,083 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf1/ca58d88083af480795630317bee9c3da 2024-11-24T15:27:54,083 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,083 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,084 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region bfa80e75ea1b7758b9c599d2ce0ef545 columnFamilyName cf2 2024-11-24T15:27:54,084 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,091 DEBUG [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/cf2/5988c863654a4b1fbfe3f47705526a5b 2024-11-24T15:27:54,091 INFO [StoreOpener-bfa80e75ea1b7758b9c599d2ce0ef545-1 {}] regionserver.HStore(327): Store=bfa80e75ea1b7758b9c599d2ce0ef545/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,091 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,092 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,093 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,094 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,094 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,095 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-24T15:27:54,097 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,098 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened bfa80e75ea1b7758b9c599d2ce0ef545; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60085083, jitterRate=-0.10466249287128448}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-24T15:27:54,098 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:27:54,099 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for bfa80e75ea1b7758b9c599d2ce0ef545: Running coprocessor pre-open hook at 1732462074075Writing region info on filesystem at 1732462074075Initializing all the Stores at 1732462074076 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074076Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074076Cleaning up temporary data from old regions at 1732462074095 (+19 ms)Running coprocessor post-open hooks at 1732462074098 (+3 ms)Region opened successfully at 1732462074099 (+1 ms) 2024-11-24T15:27:54,100 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., pid=17, masterSystemTime=1732462074070 2024-11-24T15:27:54,102 DEBUG [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:54,102 INFO [RS_OPEN_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:27:54,103 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=bfa80e75ea1b7758b9c599d2ce0ef545, regionState=OPEN, openSeqNum=18, regionLocation=c99348da3e93,38975,1732462055523 2024-11-24T15:27:54,105 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 because future has completed 2024-11-24T15:27:54,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-24T15:27:54,109 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure bfa80e75ea1b7758b9c599d2ce0ef545, server=c99348da3e93,38975,1732462055523 in 189 msec 2024-11-24T15:27:54,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-24T15:27:54,111 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=bfa80e75ea1b7758b9c599d2ce0ef545, ASSIGN in 349 msec 2024-11-24T15:27:54,111 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server c99348da3e93,41447,1732462055679 after splitting done 2024-11-24T15:27:54,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure c99348da3e93,41447,1732462055679, splitWal=true, meta=false in 608 msec 2024-11-24T15:27:54,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545., hostname=c99348da3e93,38975,1732462055523, seqNum=18] 2024-11-24T15:27:54,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=409 (was 413), OpenFileDescriptor=1051 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 228), ProcessCount=11 (was 11), AvailableMemoryMB=6398 (was 6432) 2024-11-24T15:27:54,445 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T15:27:54,457 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=6397 2024-11-24T15:27:54,457 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-24T15:27:54,472 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:54,474 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:54,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:54,477 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-72143512, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-72143512, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:54,490 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-72143512/hregion-72143512.1732462074478, exclude list is [], retry=0 2024-11-24T15:27:54,493 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:54,493 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:54,493 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:54,521 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-72143512/hregion-72143512.1732462074478 2024-11-24T15:27:54,521 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:54,522 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 65839ea9d7593ba3a2e2d1c1e3a42271, NAME => 'testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741893_1071 (size=67) 2024-11-24T15:27:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741893_1071 (size=67) 2024-11-24T15:27:54,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741893_1071 (size=67) 2024-11-24T15:27:54,534 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:54,535 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,537 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName a 2024-11-24T15:27:54,537 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,538 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,538 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,539 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName b 2024-11-24T15:27:54,539 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,539 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,540 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,541 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName c 2024-11-24T15:27:54,541 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,541 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,541 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,542 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,542 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,543 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,543 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,544 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:54,545 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,547 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:54,547 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65839ea9d7593ba3a2e2d1c1e3a42271; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66258728, jitterRate=-0.012668013572692871}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65839ea9d7593ba3a2e2d1c1e3a42271: Writing region info on filesystem at 1732462074534Initializing all the Stores at 1732462074535 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074535Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074535Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074535Cleaning up temporary data from old regions at 1732462074543 (+8 ms)Region opened successfully at 1732462074548 (+5 ms) 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 65839ea9d7593ba3a2e2d1c1e3a42271, disabling compactions & flushes 2024-11-24T15:27:54,548 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. after waiting 0 ms 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,548 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,548 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 65839ea9d7593ba3a2e2d1c1e3a42271: Waiting for close lock at 1732462074548Disabling compacts and flushes for region at 1732462074548Disabling writes for close at 1732462074548Writing region close event to WAL at 1732462074548Closed at 1732462074548 2024-11-24T15:27:54,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741892_1070 (size=175) 2024-11-24T15:27:54,552 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-72143512/hregion-72143512.1732462074478 not finished, retry = 0 2024-11-24T15:27:54,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741892_1070 (size=175) 2024-11-24T15:27:54,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741892_1070 (size=175) 2024-11-24T15:27:54,655 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:54,655 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-72143512:(num 1732462074478) 2024-11-24T15:27:54,655 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:54,657 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:54,671 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, exclude list is [], retry=0 2024-11-24T15:27:54,673 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:54,674 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:54,674 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:54,701 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 2024-11-24T15:27:54,701 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 65839ea9d7593ba3a2e2d1c1e3a42271, NAME => 'testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,702 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,703 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,704 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName a 2024-11-24T15:27:54,704 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,705 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,705 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,706 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName b 2024-11-24T15:27:54,706 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,707 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,707 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,707 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName c 2024-11-24T15:27:54,707 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:54,708 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:54,708 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,709 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,710 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,711 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,711 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,712 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:54,713 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:54,714 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65839ea9d7593ba3a2e2d1c1e3a42271; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72418812, jitterRate=0.07912439107894897}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:54,714 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65839ea9d7593ba3a2e2d1c1e3a42271: Writing region info on filesystem at 1732462074702Initializing all the Stores at 1732462074703 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074703Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074703Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462074703Cleaning up temporary data from old regions at 1732462074711 (+8 ms)Region opened successfully at 1732462074714 (+3 ms) 2024-11-24T15:27:54,741 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65839ea9d7593ba3a2e2d1c1e3a42271 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T15:27:54,759 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/a/665ea26d90a44fa2a81de5fb8db3396a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732462074715/Put/seqid=0 2024-11-24T15:27:54,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741895_1073 (size=5958) 2024-11-24T15:27:54,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741895_1073 (size=5958) 2024-11-24T15:27:54,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741895_1073 (size=5958) 2024-11-24T15:27:54,773 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/a/665ea26d90a44fa2a81de5fb8db3396a 2024-11-24T15:27:54,800 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/7e8d42fda3e54d75898a05f695eaaae2 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732462074723/Put/seqid=0 2024-11-24T15:27:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741896_1074 (size=5958) 2024-11-24T15:27:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741896_1074 (size=5958) 2024-11-24T15:27:54,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741896_1074 (size=5958) 2024-11-24T15:27:54,811 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/7e8d42fda3e54d75898a05f695eaaae2 2024-11-24T15:27:54,841 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/c/cf7f10f41ffa4936a69b9e1df9357e86 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732462074730/Put/seqid=0 2024-11-24T15:27:54,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741897_1075 (size=5958) 2024-11-24T15:27:54,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741897_1075 (size=5958) 2024-11-24T15:27:54,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/c/cf7f10f41ffa4936a69b9e1df9357e86 2024-11-24T15:27:54,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741897_1075 (size=5958) 2024-11-24T15:27:54,862 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/a/665ea26d90a44fa2a81de5fb8db3396a as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/a/665ea26d90a44fa2a81de5fb8db3396a 2024-11-24T15:27:54,868 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/a/665ea26d90a44fa2a81de5fb8db3396a, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T15:27:54,870 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/7e8d42fda3e54d75898a05f695eaaae2 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/7e8d42fda3e54d75898a05f695eaaae2 2024-11-24T15:27:54,875 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/7e8d42fda3e54d75898a05f695eaaae2, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T15:27:54,877 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/c/cf7f10f41ffa4936a69b9e1df9357e86 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/c/cf7f10f41ffa4936a69b9e1df9357e86 2024-11-24T15:27:54,884 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/c/cf7f10f41ffa4936a69b9e1df9357e86, entries=10, sequenceid=33, filesize=5.8 K 2024-11-24T15:27:54,885 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 65839ea9d7593ba3a2e2d1c1e3a42271 in 144ms, sequenceid=33, compaction requested=false 2024-11-24T15:27:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 65839ea9d7593ba3a2e2d1c1e3a42271: 2024-11-24T15:27:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 65839ea9d7593ba3a2e2d1c1e3a42271, disabling compactions & flushes 2024-11-24T15:27:54,886 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. after waiting 0 ms 2024-11-24T15:27:54,886 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,887 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. 2024-11-24T15:27:54,887 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 65839ea9d7593ba3a2e2d1c1e3a42271: Waiting for close lock at 1732462074886Disabling compacts and flushes for region at 1732462074886Disabling writes for close at 1732462074886Writing region close event to WAL at 1732462074887 (+1 ms)Closed at 1732462074887 2024-11-24T15:27:54,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741894_1072 (size=6311) 2024-11-24T15:27:54,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741894_1072 (size=6311) 2024-11-24T15:27:54,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741894_1072 (size=6311) 2024-11-24T15:27:54,898 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/7e8d42fda3e54d75898a05f695eaaae2 to hdfs://localhost:44579/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/7e8d42fda3e54d75898a05f695eaaae2 2024-11-24T15:27:54,916 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, size=6.2 K (6311bytes) 2024-11-24T15:27:54,916 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 2024-11-24T15:27:54,917 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 after 1ms 2024-11-24T15:27:54,952 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:54,952 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 took 36ms 2024-11-24T15:27:54,956 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 so closing down 2024-11-24T15:27:54,956 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:27:54,957 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732462074657.temp 2024-11-24T15:27:54,984 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp 2024-11-24T15:27:54,985 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:27:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741898_1076 (size=5575) 2024-11-24T15:27:54,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741898_1076 (size=5575) 2024-11-24T15:27:54,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741898_1076 (size=5575) 2024-11-24T15:27:54,994 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T15:27:54,996 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 2024-11-24T15:27:54,996 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-11-24T15:27:54,996 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, journal: Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, size=6.2 K (6311bytes) at 1732462074916Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 so closing down at 1732462074956 (+40 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp at 1732462074985 (+29 ms)3 split writer threads finished at 1732462074985Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732462074994 (+9 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000003-wal.1732462074657.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 at 1732462074996 (+2 ms)Processed 32 edits across 1 Regions in 43 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1732462074996 2024-11-24T15:27:54,998 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462074657 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462074657 2024-11-24T15:27:54,999 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 2024-11-24T15:27:54,999 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:55,001 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:55,016 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462075002, exclude list is [], retry=0 2024-11-24T15:27:55,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:55,019 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:55,020 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:55,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462075002 2024-11-24T15:27:55,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:55,048 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 65839ea9d7593ba3a2e2d1c1e3a42271, NAME => 'testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:55,048 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:55,048 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,048 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,049 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,050 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName a 2024-11-24T15:27:55,051 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,056 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/a/665ea26d90a44fa2a81de5fb8db3396a 2024-11-24T15:27:55,056 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,056 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,058 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName b 2024-11-24T15:27:55,058 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,058 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,058 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,059 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 65839ea9d7593ba3a2e2d1c1e3a42271 columnFamilyName c 2024-11-24T15:27:55,059 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,064 DEBUG [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/c/cf7f10f41ffa4936a69b9e1df9357e86 2024-11-24T15:27:55,064 INFO [StoreOpener-65839ea9d7593ba3a2e2d1c1e3a42271-1 {}] regionserver.HStore(327): Store=65839ea9d7593ba3a2e2d1c1e3a42271/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,064 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,065 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,067 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 2024-11-24T15:27:55,095 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:27:55,097 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 2024-11-24T15:27:55,097 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 65839ea9d7593ba3a2e2d1c1e3a42271 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T15:27:55,112 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/c63683bf200a4f5891e3c4ae42317980 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732462074723/Put/seqid=0 2024-11-24T15:27:55,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741900_1078 (size=5958) 2024-11-24T15:27:55,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741900_1078 (size=5958) 2024-11-24T15:27:55,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741900_1078 (size=5958) 2024-11-24T15:27:55,122 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/c63683bf200a4f5891e3c4ae42317980 2024-11-24T15:27:55,130 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/.tmp/b/c63683bf200a4f5891e3c4ae42317980 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/c63683bf200a4f5891e3c4ae42317980 2024-11-24T15:27:55,136 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/b/c63683bf200a4f5891e3c4ae42317980, entries=10, sequenceid=32, filesize=5.8 K 2024-11-24T15:27:55,136 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 65839ea9d7593ba3a2e2d1c1e3a42271 in 39ms, sequenceid=32, compaction requested=false; wal=null 2024-11-24T15:27:55,137 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/0000000000000000032 2024-11-24T15:27:55,138 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,138 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,139 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:55,140 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 65839ea9d7593ba3a2e2d1c1e3a42271 2024-11-24T15:27:55,142 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/65839ea9d7593ba3a2e2d1c1e3a42271/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-24T15:27:55,143 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 65839ea9d7593ba3a2e2d1c1e3a42271; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61140389, jitterRate=-0.08893720805644989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:55,143 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 65839ea9d7593ba3a2e2d1c1e3a42271: Writing region info on filesystem at 1732462075048Initializing all the Stores at 1732462075049 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075049Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075049Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075049Obtaining lock to block concurrent updates at 1732462075097 (+48 ms)Preparing flush snapshotting stores in 65839ea9d7593ba3a2e2d1c1e3a42271 at 1732462075097Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1732462075097Flushing stores of testReplayEditsWrittenViaHRegion,,1732462074473.65839ea9d7593ba3a2e2d1c1e3a42271. at 1732462075097Flushing 65839ea9d7593ba3a2e2d1c1e3a42271/b: creating writer at 1732462075097Flushing 65839ea9d7593ba3a2e2d1c1e3a42271/b: appending metadata at 1732462075111 (+14 ms)Flushing 65839ea9d7593ba3a2e2d1c1e3a42271/b: closing flushed file at 1732462075111Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7494cc49: reopening flushed file at 1732462075129 (+18 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 65839ea9d7593ba3a2e2d1c1e3a42271 in 39ms, sequenceid=32, compaction requested=false; wal=null at 1732462075136 (+7 ms)Cleaning up temporary data from old regions at 1732462075138 (+2 ms)Region opened successfully at 1732462075143 (+5 ms) 2024-11-24T15:27:55,169 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=416 (was 409) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57146 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:32816 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:57036 [Waiting for operation #34] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:60910 [Waiting for operation #36] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:54344 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:54500 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1125 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=218 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=6370 (was 6397) 2024-11-24T15:27:55,169 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-24T15:27:55,181 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=416, OpenFileDescriptor=1125, MaxFileDescriptor=1048576, SystemLoadAverage=218, ProcessCount=11, AvailableMemoryMB=6369 2024-11-24T15:27:55,181 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1125 is superior to 1024 2024-11-24T15:27:55,197 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:55,199 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:27:55,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:27:55,202 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-38538240, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-38538240, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:55,215 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-38538240/hregion-38538240.1732462075202, exclude list is [], retry=0 2024-11-24T15:27:55,220 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:55,220 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:55,220 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:55,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-24T15:27:55,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:55,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-24T15:27:55,245 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:55,246 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T15:27:55,246 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:55,247 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-24T15:27:55,247 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-24T15:27:55,260 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-38538240/hregion-38538240.1732462075202 2024-11-24T15:27:55,260 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:27:55,261 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ea9eca0f9260528710c720a0df37c4fe, NAME => 'testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:27:55,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741902_1080 (size=68) 2024-11-24T15:27:55,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741902_1080 (size=68) 2024-11-24T15:27:55,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741902_1080 (size=68) 2024-11-24T15:27:55,276 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:55,277 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,278 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName a 2024-11-24T15:27:55,278 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,279 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,279 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,280 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName b 2024-11-24T15:27:55,280 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,281 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,281 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,282 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName c 2024-11-24T15:27:55,282 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,283 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,283 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,284 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,285 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,285 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,286 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:55,287 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,289 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:27:55,289 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9eca0f9260528710c720a0df37c4fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75259993, jitterRate=0.12146128714084625}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9eca0f9260528710c720a0df37c4fe: Writing region info on filesystem at 1732462075276Initializing all the Stores at 1732462075277 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075277Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075277Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075277Cleaning up temporary data from old regions at 1732462075285 (+8 ms)Region opened successfully at 1732462075290 (+5 ms) 2024-11-24T15:27:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea9eca0f9260528710c720a0df37c4fe, disabling compactions & flushes 2024-11-24T15:27:55,290 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:27:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:27:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. after waiting 0 ms 2024-11-24T15:27:55,290 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:27:55,291 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:27:55,291 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea9eca0f9260528710c720a0df37c4fe: Waiting for close lock at 1732462075290Disabling compacts and flushes for region at 1732462075290Disabling writes for close at 1732462075290Writing region close event to WAL at 1732462075291 (+1 ms)Closed at 1732462075291 2024-11-24T15:27:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741901_1079 (size=175) 2024-11-24T15:27:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741901_1079 (size=175) 2024-11-24T15:27:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741901_1079 (size=175) 2024-11-24T15:27:55,296 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:27:55,296 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-38538240:(num 1732462075202) 2024-11-24T15:27:55,297 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:27:55,298 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:27:55,312 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, exclude list is [], retry=0 2024-11-24T15:27:55,314 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:27:55,315 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:27:55,315 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:27:55,342 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 2024-11-24T15:27:55,342 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:27:55,394 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea9eca0f9260528710c720a0df37c4fe, NAME => 'testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:27:55,397 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,397 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:27:55,397 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,397 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,399 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,400 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName a 2024-11-24T15:27:55,401 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,401 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,401 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,402 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName b 2024-11-24T15:27:55,402 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,403 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,403 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,403 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName c 2024-11-24T15:27:55,404 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:27:55,404 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:27:55,404 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,405 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,406 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,407 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,407 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,408 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:27:55,409 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,410 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9eca0f9260528710c720a0df37c4fe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68722617, jitterRate=0.024046793580055237}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:27:55,410 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:27:55,411 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9eca0f9260528710c720a0df37c4fe: Running coprocessor pre-open hook at 1732462075397Writing region info on filesystem at 1732462075397Initializing all the Stores at 1732462075398 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075398Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075399 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462075399Cleaning up temporary data from old regions at 1732462075407 (+8 ms)Running coprocessor post-open hooks at 1732462075410 (+3 ms)Region opened successfully at 1732462075410 2024-11-24T15:27:55,422 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea9eca0f9260528710c720a0df37c4fe 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-24T15:27:55,424 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:56,424 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:56,940 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T15:27:57,425 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:58,426 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:27:59,427 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:00,428 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:01,428 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:02,429 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:03,430 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:03,961 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-24T15:28:04,430 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for ea9eca0f9260528710c720a0df37c4fe/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:04,431 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea9eca0f9260528710c720a0df37c4fe: 2024-11-24T15:28:04,431 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,444 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ea9eca0f9260528710c720a0df37c4fe: 2024-11-24T15:28:04,445 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-24T15:28:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ea9eca0f9260528710c720a0df37c4fe, disabling compactions & flushes 2024-11-24T15:28:04,445 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. after waiting 0 ms 2024-11-24T15:28:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,445 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,445 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. 2024-11-24T15:28:04,445 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ea9eca0f9260528710c720a0df37c4fe: Waiting for close lock at 1732462084445Running coprocessor pre-close hooks at 1732462084445Disabling compacts and flushes for region at 1732462084445Disabling writes for close at 1732462084445Writing region close event to WAL at 1732462084445Running coprocessor post-close hooks at 1732462084445Closed at 1732462084445 2024-11-24T15:28:04,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741903_1081 (size=4236) 2024-11-24T15:28:04,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741903_1081 (size=4236) 2024-11-24T15:28:04,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741903_1081 (size=4236) 2024-11-24T15:28:04,464 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, size=4.1 K (4236bytes) 2024-11-24T15:28:04,464 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 2024-11-24T15:28:04,465 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 after 0ms 2024-11-24T15:28:04,499 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:04,499 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 took 35ms 2024-11-24T15:28:04,501 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 so closing down 2024-11-24T15:28:04,501 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:28:04,503 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1732462075298.temp 2024-11-24T15:28:04,528 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp 2024-11-24T15:28:04,528 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:28:04,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741904_1082 (size=3245) 2024-11-24T15:28:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741904_1082 (size=3245) 2024-11-24T15:28:04,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741904_1082 (size=3245) 2024-11-24T15:28:04,539 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-24T15:28:04,541 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 2024-11-24T15:28:04,541 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 42 ms; skipped=3; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-11-24T15:28:04,541 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, journal: Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, size=4.1 K (4236bytes) at 1732462084464Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 so closing down at 1732462084501 (+37 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp at 1732462084528 (+27 ms)3 split writer threads finished at 1732462084529 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1732462084539 (+10 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000004-wal.1732462075298.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 at 1732462084541 (+2 ms)Processed 23 edits across 1 Regions in 42 ms; skipped=3; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1732462084541 2024-11-24T15:28:04,543 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462075298 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462075298 2024-11-24T15:28:04,544 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 2024-11-24T15:28:04,544 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:04,546 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:04,559 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462084546, exclude list is [], retry=0 2024-11-24T15:28:04,561 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:04,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:04,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:04,590 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462084546 2024-11-24T15:28:04,591 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:04,591 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ea9eca0f9260528710c720a0df37c4fe, NAME => 'testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:04,591 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,592 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:04,592 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,592 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,593 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,594 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName a 2024-11-24T15:28:04,594 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:04,595 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:04,595 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,596 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName b 2024-11-24T15:28:04,596 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:04,596 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:04,596 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,597 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ea9eca0f9260528710c720a0df37c4fe columnFamilyName c 2024-11-24T15:28:04,597 DEBUG [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:04,597 INFO [StoreOpener-ea9eca0f9260528710c720a0df37c4fe-1 {}] regionserver.HStore(327): Store=ea9eca0f9260528710c720a0df37c4fe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:04,597 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,598 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,600 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 2024-11-24T15:28:04,626 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:04,627 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 2024-11-24T15:28:04,628 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ea9eca0f9260528710c720a0df37c4fe 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-24T15:28:04,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/a/c4e07784649a4683a93a65687b84665b is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1732462084436/Put/seqid=0 2024-11-24T15:28:04,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741906_1084 (size=5523) 2024-11-24T15:28:04,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741906_1084 (size=5523) 2024-11-24T15:28:04,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741906_1084 (size=5523) 2024-11-24T15:28:04,650 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/a/c4e07784649a4683a93a65687b84665b 2024-11-24T15:28:04,671 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/b/24f0a5043f0e4e8a8d4eff07694608a0 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1732462084432/Put/seqid=0 2024-11-24T15:28:04,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741907_1085 (size=5524) 2024-11-24T15:28:04,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741907_1085 (size=5524) 2024-11-24T15:28:04,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741907_1085 (size=5524) 2024-11-24T15:28:04,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/b/24f0a5043f0e4e8a8d4eff07694608a0 2024-11-24T15:28:04,698 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/c/38bbde90a9494c1c970405fc3272a166 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1732462084434/Put/seqid=0 2024-11-24T15:28:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741908_1086 (size=5457) 2024-11-24T15:28:04,703 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741908_1086 (size=5457) 2024-11-24T15:28:04,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741908_1086 (size=5457) 2024-11-24T15:28:04,704 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/c/38bbde90a9494c1c970405fc3272a166 2024-11-24T15:28:04,709 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/a/c4e07784649a4683a93a65687b84665b as hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/a/c4e07784649a4683a93a65687b84665b 2024-11-24T15:28:04,714 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/a/c4e07784649a4683a93a65687b84665b, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T15:28:04,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/b/24f0a5043f0e4e8a8d4eff07694608a0 as hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/b/24f0a5043f0e4e8a8d4eff07694608a0 2024-11-24T15:28:04,720 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/b/24f0a5043f0e4e8a8d4eff07694608a0, entries=7, sequenceid=26, filesize=5.4 K 2024-11-24T15:28:04,721 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/.tmp/c/38bbde90a9494c1c970405fc3272a166 as hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/c/38bbde90a9494c1c970405fc3272a166 2024-11-24T15:28:04,726 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/c/38bbde90a9494c1c970405fc3272a166, entries=6, sequenceid=26, filesize=5.3 K 2024-11-24T15:28:04,726 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ea9eca0f9260528710c720a0df37c4fe in 99ms, sequenceid=26, compaction requested=false; wal=null 2024-11-24T15:28:04,727 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/0000000000000000026 2024-11-24T15:28:04,728 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,728 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,729 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:04,730 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,732 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsAfterAbortingFlush/ea9eca0f9260528710c720a0df37c4fe/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-24T15:28:04,733 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ea9eca0f9260528710c720a0df37c4fe; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62279021, jitterRate=-0.07197026908397675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:04,733 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for ea9eca0f9260528710c720a0df37c4fe 2024-11-24T15:28:04,733 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ea9eca0f9260528710c720a0df37c4fe: Running coprocessor pre-open hook at 1732462084592Writing region info on filesystem at 1732462084592Initializing all the Stores at 1732462084593 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462084593Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462084593Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462084593Obtaining lock to block concurrent updates at 1732462084628 (+35 ms)Preparing flush snapshotting stores in ea9eca0f9260528710c720a0df37c4fe at 1732462084628Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1732462084628Flushing stores of testReplayEditsAfterAbortingFlush,,1732462075198.ea9eca0f9260528710c720a0df37c4fe. at 1732462084628Flushing ea9eca0f9260528710c720a0df37c4fe/a: creating writer at 1732462084628Flushing ea9eca0f9260528710c720a0df37c4fe/a: appending metadata at 1732462084642 (+14 ms)Flushing ea9eca0f9260528710c720a0df37c4fe/a: closing flushed file at 1732462084642Flushing ea9eca0f9260528710c720a0df37c4fe/b: creating writer at 1732462084656 (+14 ms)Flushing ea9eca0f9260528710c720a0df37c4fe/b: appending metadata at 1732462084670 (+14 ms)Flushing ea9eca0f9260528710c720a0df37c4fe/b: closing flushed file at 1732462084670Flushing ea9eca0f9260528710c720a0df37c4fe/c: creating writer at 1732462084683 (+13 ms)Flushing ea9eca0f9260528710c720a0df37c4fe/c: appending metadata at 1732462084697 (+14 ms)Flushing ea9eca0f9260528710c720a0df37c4fe/c: closing flushed file at 1732462084697Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@b29e68c: reopening flushed file at 1732462084709 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1d6edb24: reopening flushed file at 1732462084714 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@354cff32: reopening flushed file at 1732462084720 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for ea9eca0f9260528710c720a0df37c4fe in 99ms, sequenceid=26, compaction requested=false; wal=null at 1732462084726 (+6 ms)Cleaning up temporary data from old regions at 1732462084728 (+2 ms)Running coprocessor post-open hooks at 1732462084733 (+5 ms)Region opened successfully at 1732462084733 2024-11-24T15:28:04,782 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=417 (was 416) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43414 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:45848 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43380 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50226 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:45862 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50234 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1125) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 218), ProcessCount=11 (was 11), AvailableMemoryMB=6316 (was 6369) 2024-11-24T15:28:04,782 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T15:28:04,794 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=417, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=6315 2024-11-24T15:28:04,794 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-24T15:28:04,810 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:04,812 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:04,812 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:28:04,814 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60649779, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-60649779, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:04,827 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60649779/hregion-60649779.1732462084814, exclude list is [], retry=0 2024-11-24T15:28:04,830 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:04,830 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:04,830 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:04,856 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60649779/hregion-60649779.1732462084814 2024-11-24T15:28:04,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:28:04,857 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9b275e29594eed254ca3ad1aa94b29b9, NAME => 'testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:28:04,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741910_1088 (size=61) 2024-11-24T15:28:04,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741910_1088 (size=61) 2024-11-24T15:28:04,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741910_1088 (size=61) 2024-11-24T15:28:04,865 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:04,867 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,868 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b275e29594eed254ca3ad1aa94b29b9 columnFamilyName a 2024-11-24T15:28:04,868 DEBUG [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:04,869 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(327): Store=9b275e29594eed254ca3ad1aa94b29b9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:04,869 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,870 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,870 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,870 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,872 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,874 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:28:04,875 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b275e29594eed254ca3ad1aa94b29b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70217407, jitterRate=0.046320900321006775}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:28:04,876 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b275e29594eed254ca3ad1aa94b29b9: Writing region info on filesystem at 1732462084865Initializing all the Stores at 1732462084866 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462084866Cleaning up temporary data from old regions at 1732462084870 (+4 ms)Region opened successfully at 1732462084876 (+6 ms) 2024-11-24T15:28:04,876 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9b275e29594eed254ca3ad1aa94b29b9, disabling compactions & flushes 2024-11-24T15:28:04,876 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,876 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,876 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. after waiting 0 ms 2024-11-24T15:28:04,876 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,876 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,877 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9b275e29594eed254ca3ad1aa94b29b9: Waiting for close lock at 1732462084876Disabling compacts and flushes for region at 1732462084876Disabling writes for close at 1732462084876Writing region close event to WAL at 1732462084876Closed at 1732462084876 2024-11-24T15:28:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741909_1087 (size=175) 2024-11-24T15:28:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741909_1087 (size=175) 2024-11-24T15:28:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741909_1087 (size=175) 2024-11-24T15:28:04,882 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:28:04,882 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60649779:(num 1732462084814) 2024-11-24T15:28:04,882 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:04,883 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:04,897 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, exclude list is [], retry=0 2024-11-24T15:28:04,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:04,899 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:04,900 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:04,926 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 2024-11-24T15:28:04,927 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:04,927 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b275e29594eed254ca3ad1aa94b29b9, NAME => 'testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:04,927 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:04,927 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,927 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,929 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,930 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b275e29594eed254ca3ad1aa94b29b9 columnFamilyName a 2024-11-24T15:28:04,930 DEBUG [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:04,930 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(327): Store=9b275e29594eed254ca3ad1aa94b29b9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:04,930 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,931 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,932 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,932 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,932 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,934 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:04,935 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b275e29594eed254ca3ad1aa94b29b9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71000228, jitterRate=0.05798584222793579}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:28:04,935 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b275e29594eed254ca3ad1aa94b29b9: Writing region info on filesystem at 1732462084927Initializing all the Stores at 1732462084928 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462084928Cleaning up temporary data from old regions at 1732462084932 (+4 ms)Region opened successfully at 1732462084935 (+3 ms) 2024-11-24T15:28:04,944 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9b275e29594eed254ca3ad1aa94b29b9, disabling compactions & flushes 2024-11-24T15:28:04,944 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,944 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,944 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. after waiting 0 ms 2024-11-24T15:28:04,944 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,944 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,944 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. 2024-11-24T15:28:04,944 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9b275e29594eed254ca3ad1aa94b29b9: Waiting for close lock at 1732462084944Disabling compacts and flushes for region at 1732462084944Disabling writes for close at 1732462084944Writing region close event to WAL at 1732462084944Closed at 1732462084944 2024-11-24T15:28:04,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741911_1089 (size=1795) 2024-11-24T15:28:04,947 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 not finished, retry = 0 2024-11-24T15:28:04,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741911_1089 (size=1795) 2024-11-24T15:28:04,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741911_1089 (size=1795) 2024-11-24T15:28:05,063 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, size=1.8 K (1795bytes) 2024-11-24T15:28:05,063 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 2024-11-24T15:28:05,064 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 after 1ms 2024-11-24T15:28:05,092 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:05,092 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 took 29ms 2024-11-24T15:28:05,094 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 so closing down 2024-11-24T15:28:05,094 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:28:05,096 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732462084884.temp 2024-11-24T15:28:05,121 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp 2024-11-24T15:28:05,121 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:28:05,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741912_1090 (size=1795) 2024-11-24T15:28:05,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741912_1090 (size=1795) 2024-11-24T15:28:05,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741912_1090 (size=1795) 2024-11-24T15:28:05,128 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-24T15:28:05,129 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp to hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 2024-11-24T15:28:05,129 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-11-24T15:28:05,130 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, journal: Splitting hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, size=1.8 K (1795bytes) at 1732462085063Finishing writing output for hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 so closing down at 1732462085094 (+31 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp at 1732462085121 (+27 ms)3 split writer threads finished at 1732462085121Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1732462085128 (+7 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000003-wal.1732462084884.temp to hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 at 1732462085129 (+1 ms)Processed 10 edits across 1 Regions in 36 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1732462085129 2024-11-24T15:28:05,131 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462084884 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462084884 2024-11-24T15:28:05,132 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 2024-11-24T15:28:05,159 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:05,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-24T15:28:05,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-24T15:28:05,447 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:05,449 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:05,464 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462085450, exclude list is [], retry=0 2024-11-24T15:28:05,466 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:05,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:05,467 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:05,494 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462085450 2024-11-24T15:28:05,495 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:05,495 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b275e29594eed254ca3ad1aa94b29b9, NAME => 'testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:05,495 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:05,495 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,495 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,498 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,499 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b275e29594eed254ca3ad1aa94b29b9 columnFamilyName a 2024-11-24T15:28:05,499 DEBUG [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,501 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(327): Store=9b275e29594eed254ca3ad1aa94b29b9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,501 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,501 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,503 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,504 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 2024-11-24T15:28:05,530 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:05,531 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 2024-11-24T15:28:05,531 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9b275e29594eed254ca3ad1aa94b29b9 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-24T15:28:05,548 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/.tmp/a/1f4d046509c54046a6a4143a5f7b1320 is 79, key is testDatalossWhenInputError/a:x0/1732462084935/Put/seqid=0 2024-11-24T15:28:05,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741914_1092 (size=5808) 2024-11-24T15:28:05,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741914_1092 (size=5808) 2024-11-24T15:28:05,555 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741914_1092 (size=5808) 2024-11-24T15:28:05,556 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/.tmp/a/1f4d046509c54046a6a4143a5f7b1320 2024-11-24T15:28:05,565 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/.tmp/a/1f4d046509c54046a6a4143a5f7b1320 as hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/a/1f4d046509c54046a6a4143a5f7b1320 2024-11-24T15:28:05,572 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/a/1f4d046509c54046a6a4143a5f7b1320, entries=10, sequenceid=12, filesize=5.7 K 2024-11-24T15:28:05,572 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9b275e29594eed254ca3ad1aa94b29b9 in 41ms, sequenceid=12, compaction requested=false; wal=null 2024-11-24T15:28:05,573 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/0000000000000000012 2024-11-24T15:28:05,573 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,573 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,576 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,578 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-24T15:28:05,579 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b275e29594eed254ca3ad1aa94b29b9; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60864157, jitterRate=-0.09305338561534882}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:28:05,579 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b275e29594eed254ca3ad1aa94b29b9: Writing region info on filesystem at 1732462085495Initializing all the Stores at 1732462085497 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085497Obtaining lock to block concurrent updates at 1732462085531 (+34 ms)Preparing flush snapshotting stores in 9b275e29594eed254ca3ad1aa94b29b9 at 1732462085531Finished memstore snapshotting testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1732462085531Flushing stores of testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9. at 1732462085531Flushing 9b275e29594eed254ca3ad1aa94b29b9/a: creating writer at 1732462085531Flushing 9b275e29594eed254ca3ad1aa94b29b9/a: appending metadata at 1732462085548 (+17 ms)Flushing 9b275e29594eed254ca3ad1aa94b29b9/a: closing flushed file at 1732462085548Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@593967a8: reopening flushed file at 1732462085564 (+16 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 9b275e29594eed254ca3ad1aa94b29b9 in 41ms, sequenceid=12, compaction requested=false; wal=null at 1732462085572 (+8 ms)Cleaning up temporary data from old regions at 1732462085573 (+1 ms)Region opened successfully at 1732462085579 (+6 ms) 2024-11-24T15:28:05,583 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9b275e29594eed254ca3ad1aa94b29b9, NAME => 'testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:05,583 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1732462084810.9b275e29594eed254ca3ad1aa94b29b9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:05,583 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,583 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,584 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,585 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9b275e29594eed254ca3ad1aa94b29b9 columnFamilyName a 2024-11-24T15:28:05,585 DEBUG [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,591 DEBUG [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/a/1f4d046509c54046a6a4143a5f7b1320 2024-11-24T15:28:05,591 INFO [StoreOpener-9b275e29594eed254ca3ad1aa94b29b9-1 {}] regionserver.HStore(327): Store=9b275e29594eed254ca3ad1aa94b29b9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,591 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,592 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,593 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,594 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,594 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,595 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9b275e29594eed254ca3ad1aa94b29b9 2024-11-24T15:28:05,597 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testDatalossWhenInputError/9b275e29594eed254ca3ad1aa94b29b9/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-24T15:28:05,598 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9b275e29594eed254ca3ad1aa94b29b9; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75052146, jitterRate=0.1183641254901886}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-24T15:28:05,599 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9b275e29594eed254ca3ad1aa94b29b9: Writing region info on filesystem at 1732462085583Initializing all the Stores at 1732462085584 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085584Cleaning up temporary data from old regions at 1732462085594 (+10 ms)Region opened successfully at 1732462085598 (+4 ms) 2024-11-24T15:28:05,616 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=427 (was 417) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43454 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50308 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50226 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:45862 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:45938 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=184 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=6309 (was 6315) 2024-11-24T15:28:05,616 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T15:28:05,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=427, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=184, ProcessCount=11, AvailableMemoryMB=6308 2024-11-24T15:28:05,629 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-24T15:28:05,643 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:05,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:05,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:28:05,648 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-89725760, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-89725760, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:05,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-89725760/hregion-89725760.1732462085648, exclude list is [], retry=0 2024-11-24T15:28:05,663 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:05,663 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:05,663 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:05,698 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-89725760/hregion-89725760.1732462085648 2024-11-24T15:28:05,699 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:28:05,699 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a72d72c861e6a45e58e39111b5c618f3, NAME => 'testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:28:05,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741916_1094 (size=63) 2024-11-24T15:28:05,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741916_1094 (size=63) 2024-11-24T15:28:05,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741916_1094 (size=63) 2024-11-24T15:28:05,708 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:05,709 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,711 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName a 2024-11-24T15:28:05,711 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,711 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,711 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,712 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName b 2024-11-24T15:28:05,712 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,713 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,713 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,714 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName c 2024-11-24T15:28:05,714 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,715 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,715 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,715 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,716 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,717 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,717 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,718 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:05,719 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,720 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:28:05,721 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a72d72c861e6a45e58e39111b5c618f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61071659, jitterRate=-0.08996136486530304}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:05,721 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a72d72c861e6a45e58e39111b5c618f3: Writing region info on filesystem at 1732462085708Initializing all the Stores at 1732462085709 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085709Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085709Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085709Cleaning up temporary data from old regions at 1732462085717 (+8 ms)Region opened successfully at 1732462085721 (+4 ms) 2024-11-24T15:28:05,721 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a72d72c861e6a45e58e39111b5c618f3, disabling compactions & flushes 2024-11-24T15:28:05,722 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:05,722 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:05,722 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. after waiting 0 ms 2024-11-24T15:28:05,722 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:05,722 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:05,722 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a72d72c861e6a45e58e39111b5c618f3: Waiting for close lock at 1732462085721Disabling compacts and flushes for region at 1732462085721Disabling writes for close at 1732462085722 (+1 ms)Writing region close event to WAL at 1732462085722Closed at 1732462085722 2024-11-24T15:28:05,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741915_1093 (size=175) 2024-11-24T15:28:05,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741915_1093 (size=175) 2024-11-24T15:28:05,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741915_1093 (size=175) 2024-11-24T15:28:05,730 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:28:05,730 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-89725760:(num 1732462085648) 2024-11-24T15:28:05,730 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:05,731 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:05,744 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, exclude list is [], retry=0 2024-11-24T15:28:05,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:05,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:05,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:05,774 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 2024-11-24T15:28:05,775 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:28:05,775 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a72d72c861e6a45e58e39111b5c618f3, NAME => 'testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:05,775 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:05,775 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,775 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,777 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,777 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName a 2024-11-24T15:28:05,778 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,778 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,778 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,779 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName b 2024-11-24T15:28:05,779 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,779 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,779 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,780 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName c 2024-11-24T15:28:05,780 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:05,780 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:05,781 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,781 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,782 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,783 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,783 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,783 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:05,785 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:05,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a72d72c861e6a45e58e39111b5c618f3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68834927, jitterRate=0.025720342993736267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:05,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a72d72c861e6a45e58e39111b5c618f3: Writing region info on filesystem at 1732462085776Initializing all the Stores at 1732462085776Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085776Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085777 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462085777Cleaning up temporary data from old regions at 1732462085783 (+6 ms)Region opened successfully at 1732462085786 (+3 ms) 2024-11-24T15:28:05,789 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1732462085789/Put/seqid=0 2024-11-24T15:28:05,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741918_1096 (size=4875) 2024-11-24T15:28:05,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741918_1096 (size=4875) 2024-11-24T15:28:05,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741918_1096 (size=4875) 2024-11-24T15:28:05,798 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1732462085798/Put/seqid=0 2024-11-24T15:28:05,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741919_1097 (size=4875) 2024-11-24T15:28:05,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741919_1097 (size=4875) 2024-11-24T15:28:05,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741919_1097 (size=4875) 2024-11-24T15:28:05,810 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1732462085810/Put/seqid=0 2024-11-24T15:28:05,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741920_1098 (size=4875) 2024-11-24T15:28:05,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741920_1098 (size=4875) 2024-11-24T15:28:05,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741920_1098 (size=4875) 2024-11-24T15:28:05,819 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,823 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-24T15:28:05,823 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T15:28:05,823 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,827 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-24T15:28:05,827 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T15:28:05,827 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,830 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-24T15:28:05,830 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-24T15:28:05,831 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a72d72c861e6a45e58e39111b5c618f3 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-24T15:28:05,847 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp/a/318e3cb9ec994eab8040fcfe61e1c666 is 55, key is testCompactedBulkLoadedFiles/a:a/1732462085786/Put/seqid=0 2024-11-24T15:28:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741921_1099 (size=5107) 2024-11-24T15:28:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741921_1099 (size=5107) 2024-11-24T15:28:05,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741921_1099 (size=5107) 2024-11-24T15:28:05,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp/a/318e3cb9ec994eab8040fcfe61e1c666 2024-11-24T15:28:05,862 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp/a/318e3cb9ec994eab8040fcfe61e1c666 as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666 2024-11-24T15:28:05,866 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666, entries=1, sequenceid=4, filesize=5.0 K 2024-11-24T15:28:05,867 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for a72d72c861e6a45e58e39111b5c618f3 in 37ms, sequenceid=4, compaction requested=false 2024-11-24T15:28:05,867 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for a72d72c861e6a45e58e39111b5c618f3: 2024-11-24T15:28:05,869 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ 2024-11-24T15:28:05,870 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ 2024-11-24T15:28:05,870 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ 2024-11-24T15:28:05,871 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile0 into a72d72c861e6a45e58e39111b5c618f3/a as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ - updating store file list. 2024-11-24T15:28:05,875 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c44f4bd06223445ab437a8483e66d852_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:05,875 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ into a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,875 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile0 into a72d72c861e6a45e58e39111b5c618f3/a (new location: hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_) 2024-11-24T15:28:05,875 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile1 into a72d72c861e6a45e58e39111b5c618f3/a as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ - updating store file list. 2024-11-24T15:28:05,879 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:05,879 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ into a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,879 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile1 into a72d72c861e6a45e58e39111b5c618f3/a (new location: hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_) 2024-11-24T15:28:05,880 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile2 into a72d72c861e6a45e58e39111b5c618f3/a as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ - updating store file list. 2024-11-24T15:28:05,884 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:05,884 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ into a72d72c861e6a45e58e39111b5c618f3/a 2024-11-24T15:28:05,884 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44579/hbase/testCompactedBulkLoadedFiles/hfile2 into a72d72c861e6a45e58e39111b5c618f3/a (new location: hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_) 2024-11-24T15:28:05,891 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-24T15:28:05,892 DEBUG [Time-limited test {}] regionserver.HStore(1541): a72d72c861e6a45e58e39111b5c618f3/a is initiating major compaction (all files) 2024-11-24T15:28:05,892 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of a72d72c861e6a45e58e39111b5c618f3/a in testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:05,892 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_] into tmpdir=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp, totalSize=19.3 K 2024-11-24T15:28:05,892 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 318e3cb9ec994eab8040fcfe61e1c666, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1732462085786 2024-11-24T15:28:05,893 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting c44f4bd06223445ab437a8483e66d852_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T15:28:05,893 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T15:28:05,893 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-24T15:28:05,904 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp/a/5fc8d9559aa14b55b5bfa22d80bfcd1a is 55, key is testCompactedBulkLoadedFiles/a:a/1732462085786/Put/seqid=0 2024-11-24T15:28:05,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741922_1100 (size=6154) 2024-11-24T15:28:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741922_1100 (size=6154) 2024-11-24T15:28:05,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741922_1100 (size=6154) 2024-11-24T15:28:05,918 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/.tmp/a/5fc8d9559aa14b55b5bfa22d80bfcd1a as hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/5fc8d9559aa14b55b5bfa22d80bfcd1a 2024-11-24T15:28:05,923 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in a72d72c861e6a45e58e39111b5c618f3/a of a72d72c861e6a45e58e39111b5c618f3 into 5fc8d9559aa14b55b5bfa22d80bfcd1a(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-24T15:28:05,923 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for a72d72c861e6a45e58e39111b5c618f3: 2024-11-24T15:28:05,924 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T15:28:05,924 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-24T15:28:05,955 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, size=0 (0bytes) 2024-11-24T15:28:05,955 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 might be still open, length is 0 2024-11-24T15:28:05,955 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 2024-11-24T15:28:05,956 WARN [IPC Server handler 1 on default port 44579 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-24T15:28:05,956 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 after 1ms 2024-11-24T15:28:06,760 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:45970 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:34483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:45970 dst: /127.0.0.1:34483 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34483 remote=/127.0.0.1:45970]. Total timeout mills is 60000, 59163 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:06,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43502 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:39259:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43502 dst: /127.0.0.1:39259 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:06,761 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50340 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50340 dst: /127.0.0.1:41867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:06,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741917_1101 (size=1794) 2024-11-24T15:28:06,768 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741917_1101 (size=1794) 2024-11-24T15:28:06,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741917_1101 (size=1794) 2024-11-24T15:28:09,957 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 after 4002ms 2024-11-24T15:28:09,966 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-24T15:28:09,986 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:09,986 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 took 4031ms 2024-11-24T15:28:09,988 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732; continuing. 2024-11-24T15:28:09,988 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 so closing down 2024-11-24T15:28:09,988 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:28:09,990 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732462085732.temp 2024-11-24T15:28:10,020 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp 2024-11-24T15:28:10,020 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:28:10,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741923_1102 (size=800) 2024-11-24T15:28:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741923_1102 (size=800) 2024-11-24T15:28:10,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741923_1102 (size=800) 2024-11-24T15:28:10,029 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-24T15:28:10,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp to hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 2024-11-24T15:28:10,030 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 43 ms; skipped=3; WAL=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T15:28:10,030 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, journal: Splitting hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, size=0 (0bytes) at 1732462085955Finishing writing output for hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 so closing down at 1732462089988 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp at 1732462090020 (+32 ms)3 split writer threads finished at 1732462090020Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1732462090029 (+9 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000003-wal.1732462085732.temp to hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 at 1732462090030 (+1 ms)Processed 5 edits across 1 Regions in 43 ms; skipped=3; WAL=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732, size=0, length=0, corrupted=false, cancelled=false at 1732462090030 2024-11-24T15:28:10,032 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462085732 2024-11-24T15:28:10,033 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 2024-11-24T15:28:10,033 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:10,035 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:10,049 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462090035, exclude list is [], retry=0 2024-11-24T15:28:10,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:10,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:10,051 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:10,079 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462090035 2024-11-24T15:28:10,079 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:10,079 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a72d72c861e6a45e58e39111b5c618f3, NAME => 'testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:10,079 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:10,080 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,080 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,081 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,082 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName a 2024-11-24T15:28:10,082 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,089 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666 2024-11-24T15:28:10,092 DEBUG [StoreFileOpener-a72d72c861e6a45e58e39111b5c618f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:10,093 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ 2024-11-24T15:28:10,096 DEBUG [StoreFileOpener-a72d72c861e6a45e58e39111b5c618f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:10,097 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ 2024-11-24T15:28:10,100 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/5fc8d9559aa14b55b5bfa22d80bfcd1a 2024-11-24T15:28:10,104 DEBUG [StoreFileOpener-a72d72c861e6a45e58e39111b5c618f3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c44f4bd06223445ab437a8483e66d852_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-24T15:28:10,104 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ 2024-11-24T15:28:10,104 WARN [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-24T15:28:10,104 WARN [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-24T15:28:10,104 WARN [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-24T15:28:10,104 WARN [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@7a4d7efe 2024-11-24T15:28:10,104 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_] to archive 2024-11-24T15:28:10,105 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-24T15:28:10,107 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666 to hdfs://localhost:44579/hbase/archive/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/318e3cb9ec994eab8040fcfe61e1c666 2024-11-24T15:28:10,108 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ to hdfs://localhost:44579/hbase/archive/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_ 2024-11-24T15:28:10,109 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ to hdfs://localhost:44579/hbase/archive/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_ 2024-11-24T15:28:10,110 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ to hdfs://localhost:44579/hbase/archive/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/a/c44f4bd06223445ab437a8483e66d852_SeqId_4_ 2024-11-24T15:28:10,110 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,110 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,111 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName b 2024-11-24T15:28:10,111 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,111 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,112 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,112 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a72d72c861e6a45e58e39111b5c618f3 columnFamilyName c 2024-11-24T15:28:10,112 DEBUG [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,113 INFO [StoreOpener-a72d72c861e6a45e58e39111b5c618f3-1 {}] regionserver.HStore(327): Store=a72d72c861e6a45e58e39111b5c618f3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,113 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,114 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,115 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 2024-11-24T15:28:10,144 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:10,146 DEBUG [Time-limited test {}] regionserver.HRegion(5836): a72d72c861e6a45e58e39111b5c618f3 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "a72d72c861e6a45e58e39111b5c618f3" family_name: "a" compaction_input: "318e3cb9ec994eab8040fcfe61e1c666" compaction_input: "c44f4bd06223445ab437a8483e66d852_SeqId_4_" compaction_input: "32d0f009cf6f49e4a2b00eeded7517bd_SeqId_4_" compaction_input: "4c4f984f01ad4f859ea62d0c0a66ee61_SeqId_4_" compaction_output: "5fc8d9559aa14b55b5bfa22d80bfcd1a" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-24T15:28:10,146 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-24T15:28:10,146 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 2024-11-24T15:28:10,147 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/0000000000000000008 2024-11-24T15:28:10,148 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,148 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,149 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:10,150 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a72d72c861e6a45e58e39111b5c618f3 2024-11-24T15:28:10,152 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testCompactedBulkLoadedFiles/a72d72c861e6a45e58e39111b5c618f3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-24T15:28:10,153 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a72d72c861e6a45e58e39111b5c618f3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59556894, jitterRate=-0.11253312230110168}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:10,154 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a72d72c861e6a45e58e39111b5c618f3: Writing region info on filesystem at 1732462090080Initializing all the Stores at 1732462090080Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090081 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090081Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090081Cleaning up temporary data from old regions at 1732462090148 (+67 ms)Region opened successfully at 1732462090153 (+5 ms) 2024-11-24T15:28:10,156 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a72d72c861e6a45e58e39111b5c618f3, disabling compactions & flushes 2024-11-24T15:28:10,156 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:10,156 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:10,156 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. after waiting 0 ms 2024-11-24T15:28:10,156 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:10,157 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1732462085644.a72d72c861e6a45e58e39111b5c618f3. 2024-11-24T15:28:10,157 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a72d72c861e6a45e58e39111b5c618f3: Waiting for close lock at 1732462090156Disabling compacts and flushes for region at 1732462090156Disabling writes for close at 1732462090156Writing region close event to WAL at 1732462090157 (+1 ms)Closed at 1732462090157 2024-11-24T15:28:10,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741924_1103 (size=175) 2024-11-24T15:28:10,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741924_1103 (size=175) 2024-11-24T15:28:10,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741924_1103 (size=175) 2024-11-24T15:28:10,162 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:28:10,162 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732462090035) 2024-11-24T15:28:10,178 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=441 (was 427) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44579 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_116547971_22 at /127.0.0.1:50404 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_116547971_22 at /127.0.0.1:46024 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_116547971_22 at /127.0.0.1:43564 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=169 (was 184), ProcessCount=11 (was 11), AvailableMemoryMB=6301 (was 6308) 2024-11-24T15:28:10,178 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T15:28:10,194 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=441, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=169, ProcessCount=11, AvailableMemoryMB=6300 2024-11-24T15:28:10,194 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-24T15:28:10,214 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:10,216 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-24T15:28:10,216 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-24T15:28:10,219 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-92467938, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/hregion-92467938, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:10,235 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-92467938/hregion-92467938.1732462090219, exclude list is [], retry=0 2024-11-24T15:28:10,238 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:10,238 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:10,239 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:10,270 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-92467938/hregion-92467938.1732462090219 2024-11-24T15:28:10,271 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:10,271 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5f473e62c97758cd46c46a7e69adefcd, NAME => 'testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44579/hbase 2024-11-24T15:28:10,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741926_1105 (size=67) 2024-11-24T15:28:10,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741926_1105 (size=67) 2024-11-24T15:28:10,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741926_1105 (size=67) 2024-11-24T15:28:10,280 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:10,282 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,283 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName a 2024-11-24T15:28:10,283 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,284 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,284 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,285 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName b 2024-11-24T15:28:10,285 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,286 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,286 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,287 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName c 2024-11-24T15:28:10,287 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,287 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,287 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,288 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,289 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,289 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,290 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:10,291 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,293 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-24T15:28:10,294 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f473e62c97758cd46c46a7e69adefcd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73254637, jitterRate=0.09157915413379669}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f473e62c97758cd46c46a7e69adefcd: Writing region info on filesystem at 1732462090281Initializing all the Stores at 1732462090281Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090281Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090282 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090282Cleaning up temporary data from old regions at 1732462090289 (+7 ms)Region opened successfully at 1732462090294 (+5 ms) 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f473e62c97758cd46c46a7e69adefcd, disabling compactions & flushes 2024-11-24T15:28:10,294 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. after waiting 0 ms 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,294 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,294 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f473e62c97758cd46c46a7e69adefcd: Waiting for close lock at 1732462090294Disabling compacts and flushes for region at 1732462090294Disabling writes for close at 1732462090294Writing region close event to WAL at 1732462090294Closed at 1732462090294 2024-11-24T15:28:10,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741925_1104 (size=175) 2024-11-24T15:28:10,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741925_1104 (size=175) 2024-11-24T15:28:10,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741925_1104 (size=175) 2024-11-24T15:28:10,298 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:28:10,298 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-92467938:(num 1732462090219) 2024-11-24T15:28:10,299 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:10,300 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:10,316 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, exclude list is [], retry=0 2024-11-24T15:28:10,319 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:10,319 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:10,320 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:10,348 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 2024-11-24T15:28:10,348 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739)] 2024-11-24T15:28:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f473e62c97758cd46c46a7e69adefcd, NAME => 'testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,349 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,350 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,351 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName a 2024-11-24T15:28:10,351 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,352 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,352 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,353 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName b 2024-11-24T15:28:10,353 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,353 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,353 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,354 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName c 2024-11-24T15:28:10,354 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,354 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,355 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,355 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,356 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,357 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,358 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,358 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:10,360 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,361 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f473e62c97758cd46c46a7e69adefcd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70320944, jitterRate=0.04786372184753418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:10,361 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f473e62c97758cd46c46a7e69adefcd: Writing region info on filesystem at 1732462090349Initializing all the Stores at 1732462090350 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090350Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090350Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090350Cleaning up temporary data from old regions at 1732462090358 (+8 ms)Region opened successfully at 1732462090361 (+3 ms) 2024-11-24T15:28:10,369 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f473e62c97758cd46c46a7e69adefcd 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-24T15:28:10,389 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/18569931486a46649cfe7d859a99d37a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1732462090361/Put/seqid=0 2024-11-24T15:28:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741928_1107 (size=5958) 2024-11-24T15:28:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741928_1107 (size=5958) 2024-11-24T15:28:10,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741928_1107 (size=5958) 2024-11-24T15:28:10,397 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/18569931486a46649cfe7d859a99d37a 2024-11-24T15:28:10,402 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/18569931486a46649cfe7d859a99d37a as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/18569931486a46649cfe7d859a99d37a 2024-11-24T15:28:10,407 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/18569931486a46649cfe7d859a99d37a, entries=10, sequenceid=13, filesize=5.8 K 2024-11-24T15:28:10,409 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 5f473e62c97758cd46c46a7e69adefcd in 39ms, sequenceid=13, compaction requested=false 2024-11-24T15:28:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 5f473e62c97758cd46c46a7e69adefcd: 2024-11-24T15:28:10,429 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f473e62c97758cd46c46a7e69adefcd, disabling compactions & flushes 2024-11-24T15:28:10,429 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,429 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,429 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. after waiting 0 ms 2024-11-24T15:28:10,429 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,430 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,430 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:10,430 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f473e62c97758cd46c46a7e69adefcd: Waiting for close lock at 1732462090429Disabling compacts and flushes for region at 1732462090429Disabling writes for close at 1732462090429Writing region close event to WAL at 1732462090430 (+1 ms)Closed at 1732462090430 2024-11-24T15:28:10,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741927_1106 (size=6243) 2024-11-24T15:28:10,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741927_1106 (size=6243) 2024-11-24T15:28:10,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741927_1106 (size=6243) 2024-11-24T15:28:10,449 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, size=6.1 K (6243bytes) 2024-11-24T15:28:10,449 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 2024-11-24T15:28:10,450 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 after 1ms 2024-11-24T15:28:10,478 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:10,479 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 took 29ms 2024-11-24T15:28:10,481 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 so closing down 2024-11-24T15:28:10,481 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:28:10,483 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1732462090301.temp 2024-11-24T15:28:10,518 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp 2024-11-24T15:28:10,519 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:28:10,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741929_1108 (size=5575) 2024-11-24T15:28:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741929_1108 (size=5575) 2024-11-24T15:28:10,527 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741929_1108 (size=5575) 2024-11-24T15:28:10,527 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T15:28:10,529 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 2024-11-24T15:28:10,529 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-11-24T15:28:10,529 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, journal: Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, size=6.1 K (6243bytes) at 1732462090449Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 so closing down at 1732462090481 (+32 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp at 1732462090518 (+37 ms)3 split writer threads finished at 1732462090519 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732462090528 (+9 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000003-wal.1732462090301.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 at 1732462090529 (+1 ms)Processed 32 edits across 1 Regions in 50 ms; skipped=2; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1732462090529 2024-11-24T15:28:10,531 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090301 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462090301 2024-11-24T15:28:10,532 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 2024-11-24T15:28:10,532 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:10,533 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:10,548 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, exclude list is [], retry=0 2024-11-24T15:28:10,550 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:10,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:10,551 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:10,579 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 2024-11-24T15:28:10,579 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:39897:39897),(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973)] 2024-11-24T15:28:10,579 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 5f473e62c97758cd46c46a7e69adefcd, NAME => 'testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.', STARTKEY => '', ENDKEY => ''} 2024-11-24T15:28:10,579 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:10,580 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,580 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,581 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,582 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName a 2024-11-24T15:28:10,582 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,588 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/18569931486a46649cfe7d859a99d37a 2024-11-24T15:28:10,588 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,588 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,589 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName b 2024-11-24T15:28:10,589 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,589 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,589 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,590 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName c 2024-11-24T15:28:10,590 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:10,590 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:10,590 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,591 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,592 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,593 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 2024-11-24T15:28:10,622 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:10,623 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 2024-11-24T15:28:10,624 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f473e62c97758cd46c46a7e69adefcd 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-24T15:28:10,640 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/9d87ed50fc324b6a8d6303a82c205afe is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1732462090409/Put/seqid=0 2024-11-24T15:28:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741931_1110 (size=5958) 2024-11-24T15:28:10,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741931_1110 (size=5958) 2024-11-24T15:28:10,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741931_1110 (size=5958) 2024-11-24T15:28:10,647 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/9d87ed50fc324b6a8d6303a82c205afe 2024-11-24T15:28:10,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/40dbfa77bda34740bc79250d2c1f99e4 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1732462090417/Put/seqid=0 2024-11-24T15:28:10,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741932_1111 (size=5958) 2024-11-24T15:28:10,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741932_1111 (size=5958) 2024-11-24T15:28:10,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741932_1111 (size=5958) 2024-11-24T15:28:10,675 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/40dbfa77bda34740bc79250d2c1f99e4 2024-11-24T15:28:10,681 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/9d87ed50fc324b6a8d6303a82c205afe as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/b/9d87ed50fc324b6a8d6303a82c205afe 2024-11-24T15:28:10,686 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/b/9d87ed50fc324b6a8d6303a82c205afe, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T15:28:10,687 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/40dbfa77bda34740bc79250d2c1f99e4 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/c/40dbfa77bda34740bc79250d2c1f99e4 2024-11-24T15:28:10,692 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/c/40dbfa77bda34740bc79250d2c1f99e4, entries=10, sequenceid=35, filesize=5.8 K 2024-11-24T15:28:10,692 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 5f473e62c97758cd46c46a7e69adefcd in 69ms, sequenceid=35, compaction requested=false; wal=null 2024-11-24T15:28:10,693 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000035 2024-11-24T15:28:10,694 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,694 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,695 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:10,696 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:10,698 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-24T15:28:10,699 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f473e62c97758cd46c46a7e69adefcd; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68669391, jitterRate=0.0232536643743515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:10,699 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f473e62c97758cd46c46a7e69adefcd: Writing region info on filesystem at 1732462090580Initializing all the Stores at 1732462090581 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090581Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090581Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462090581Obtaining lock to block concurrent updates at 1732462090624 (+43 ms)Preparing flush snapshotting stores in 5f473e62c97758cd46c46a7e69adefcd at 1732462090624Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1732462090624Flushing stores of testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. at 1732462090624Flushing 5f473e62c97758cd46c46a7e69adefcd/b: creating writer at 1732462090624Flushing 5f473e62c97758cd46c46a7e69adefcd/b: appending metadata at 1732462090639 (+15 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/b: closing flushed file at 1732462090639Flushing 5f473e62c97758cd46c46a7e69adefcd/c: creating writer at 1732462090652 (+13 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/c: appending metadata at 1732462090667 (+15 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/c: closing flushed file at 1732462090667Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e7c757d: reopening flushed file at 1732462090680 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@797a22b6: reopening flushed file at 1732462090687 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 5f473e62c97758cd46c46a7e69adefcd in 69ms, sequenceid=35, compaction requested=false; wal=null at 1732462090692 (+5 ms)Cleaning up temporary data from old regions at 1732462090694 (+2 ms)Region opened successfully at 1732462090699 (+5 ms) 2024-11-24T15:28:10,777 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, size=0 (0bytes) 2024-11-24T15:28:10,777 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 might be still open, length is 0 2024-11-24T15:28:10,777 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 2024-11-24T15:28:10,777 WARN [IPC Server handler 3 on default port 44579 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-24T15:28:10,777 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 after 0ms 2024-11-24T15:28:12,770 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:43626 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:39259:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43626 dst: /127.0.0.1:39259 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:39259 remote=/127.0.0.1:43626]. Total timeout mills is 60000, 57959 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:12,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:50456 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41867:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50456 dst: /127.0.0.1:41867 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:12,771 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_460530494_22 at /127.0.0.1:46102 [Receiving block BP-532991956-172.17.0.2-1732462051681:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:34483:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46102 dst: /127.0.0.1:34483 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:12,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741930_1112 (size=5567) 2024-11-24T15:28:12,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741930_1112 (size=5567) 2024-11-24T15:28:14,778 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 after 4001ms 2024-11-24T15:28:14,807 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:14,807 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 took 4031ms 2024-11-24T15:28:14,809 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534; continuing. 2024-11-24T15:28:14,809 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 so closing down 2024-11-24T15:28:14,809 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-24T15:28:14,810 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1732462090534.temp 2024-11-24T15:28:14,837 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp 2024-11-24T15:28:14,837 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-24T15:28:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741933_1113 (size=5575) 2024-11-24T15:28:14,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741933_1113 (size=5575) 2024-11-24T15:28:14,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741933_1113 (size=5575) 2024-11-24T15:28:14,844 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-24T15:28:14,846 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 2024-11-24T15:28:14,846 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, size=0, length=0, corrupted=false, cancelled=false 2024-11-24T15:28:14,846 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, journal: Splitting hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, size=0 (0bytes) at 1732462090777Finishing writing output for hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 so closing down at 1732462094809 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp at 1732462094837 (+28 ms)3 split writer threads finished at 1732462094837Closed recovered edits writer path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1732462094844 (+7 ms)Rename recovered edits hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000037-wal.1732462090534.temp to hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 at 1732462094846 (+2 ms)Processed 30 edits across 1 Regions in 39 ms; skipped=0; WAL=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534, size=0, length=0, corrupted=false, cancelled=false at 1732462094846 2024-11-24T15:28:14,847 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 to hdfs://localhost:44579/hbase/oldWALs/wal.1732462090534 2024-11-24T15:28:14,848 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 2024-11-24T15:28:14,848 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-24T15:28:14,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44579/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213, archiveDir=hdfs://localhost:44579/hbase/oldWALs, maxLogs=32 2024-11-24T15:28:14,868 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462094850, exclude list is [], retry=0 2024-11-24T15:28:14,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41867,DS-0d56fc79-22a0-4846-addc-31a51dc0fa87,DISK] 2024-11-24T15:28:14,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-7326fdfb-1211-4980-9f90-113141d326d8,DISK] 2024-11-24T15:28:14,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:39259,DS-432632f3-225d-41fa-b045-26dfe1da38ff,DISK] 2024-11-24T15:28:14,896 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462094850 2024-11-24T15:28:14,897 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:43739:43739),(127.0.0.1/127.0.0.1:43973:43973),(127.0.0.1/127.0.0.1:39897:39897)] 2024-11-24T15:28:14,897 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-24T15:28:14,898 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,899 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName a 2024-11-24T15:28:14,899 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:14,904 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/18569931486a46649cfe7d859a99d37a 2024-11-24T15:28:14,904 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:14,904 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,905 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName b 2024-11-24T15:28:14,905 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:14,909 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/b/9d87ed50fc324b6a8d6303a82c205afe 2024-11-24T15:28:14,909 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:14,909 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,910 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5f473e62c97758cd46c46a7e69adefcd columnFamilyName c 2024-11-24T15:28:14,910 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-24T15:28:14,914 DEBUG [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/c/40dbfa77bda34740bc79250d2c1f99e4 2024-11-24T15:28:14,914 INFO [StoreOpener-5f473e62c97758cd46c46a7e69adefcd-1 {}] regionserver.HStore(327): Store=5f473e62c97758cd46c46a7e69adefcd/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-24T15:28:14,915 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,915 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,916 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:14,917 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 2024-11-24T15:28:14,943 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-11-24T15:28:14,948 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 2024-11-24T15:28:14,948 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5f473e62c97758cd46c46a7e69adefcd 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-24T15:28:14,963 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/7c1bfe7b2be544b6b8df366fae9ed9e4 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1732462090706/Put/seqid=0 2024-11-24T15:28:14,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741935_1115 (size=5958) 2024-11-24T15:28:14,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741935_1115 (size=5958) 2024-11-24T15:28:14,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741935_1115 (size=5958) 2024-11-24T15:28:14,971 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/7c1bfe7b2be544b6b8df366fae9ed9e4 2024-11-24T15:28:14,990 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/dce5b4746bb24b2da9db8eac60e2a00f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1732462090714/Put/seqid=0 2024-11-24T15:28:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741936_1116 (size=5958) 2024-11-24T15:28:14,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741936_1116 (size=5958) 2024-11-24T15:28:14,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741936_1116 (size=5958) 2024-11-24T15:28:14,997 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/dce5b4746bb24b2da9db8eac60e2a00f 2024-11-24T15:28:15,017 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/c3c6688e3bca491b97109035347b9656 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1732462090722/Put/seqid=0 2024-11-24T15:28:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741937_1117 (size=5958) 2024-11-24T15:28:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741937_1117 (size=5958) 2024-11-24T15:28:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741937_1117 (size=5958) 2024-11-24T15:28:15,024 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/c3c6688e3bca491b97109035347b9656 2024-11-24T15:28:15,029 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/a/7c1bfe7b2be544b6b8df366fae9ed9e4 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/7c1bfe7b2be544b6b8df366fae9ed9e4 2024-11-24T15:28:15,033 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/a/7c1bfe7b2be544b6b8df366fae9ed9e4, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T15:28:15,034 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/b/dce5b4746bb24b2da9db8eac60e2a00f as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/b/dce5b4746bb24b2da9db8eac60e2a00f 2024-11-24T15:28:15,038 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/b/dce5b4746bb24b2da9db8eac60e2a00f, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T15:28:15,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/.tmp/c/c3c6688e3bca491b97109035347b9656 as hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/c/c3c6688e3bca491b97109035347b9656 2024-11-24T15:28:15,043 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/c/c3c6688e3bca491b97109035347b9656, entries=10, sequenceid=66, filesize=5.8 K 2024-11-24T15:28:15,043 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 5f473e62c97758cd46c46a7e69adefcd in 95ms, sequenceid=66, compaction requested=false; wal=null 2024-11-24T15:28:15,043 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/0000000000000000066 2024-11-24T15:28:15,044 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:15,045 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:15,045 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-24T15:28:15,046 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5f473e62c97758cd46c46a7e69adefcd 2024-11-24T15:28:15,048 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/hbase/data/default/testReplayEditsWrittenViaHRegion/5f473e62c97758cd46c46a7e69adefcd/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-24T15:28:15,049 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5f473e62c97758cd46c46a7e69adefcd; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63057521, jitterRate=-0.060369715094566345}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-24T15:28:15,049 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5f473e62c97758cd46c46a7e69adefcd: Writing region info on filesystem at 1732462094897Initializing all the Stores at 1732462094898 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462094898Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462094898Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732462094898Obtaining lock to block concurrent updates at 1732462094948 (+50 ms)Preparing flush snapshotting stores in 5f473e62c97758cd46c46a7e69adefcd at 1732462094948Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1732462094948Flushing stores of testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. at 1732462094948Flushing 5f473e62c97758cd46c46a7e69adefcd/a: creating writer at 1732462094948Flushing 5f473e62c97758cd46c46a7e69adefcd/a: appending metadata at 1732462094962 (+14 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/a: closing flushed file at 1732462094962Flushing 5f473e62c97758cd46c46a7e69adefcd/b: creating writer at 1732462094976 (+14 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/b: appending metadata at 1732462094990 (+14 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/b: closing flushed file at 1732462094990Flushing 5f473e62c97758cd46c46a7e69adefcd/c: creating writer at 1732462095002 (+12 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/c: appending metadata at 1732462095016 (+14 ms)Flushing 5f473e62c97758cd46c46a7e69adefcd/c: closing flushed file at 1732462095016Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1586850e: reopening flushed file at 1732462095028 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@20358d9: reopening flushed file at 1732462095033 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ef9ab50: reopening flushed file at 1732462095038 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 5f473e62c97758cd46c46a7e69adefcd in 95ms, sequenceid=66, compaction requested=false; wal=null at 1732462095043 (+5 ms)Cleaning up temporary data from old regions at 1732462095045 (+2 ms)Region opened successfully at 1732462095049 (+4 ms) 2024-11-24T15:28:15,062 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5f473e62c97758cd46c46a7e69adefcd, disabling compactions & flushes 2024-11-24T15:28:15,062 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:15,062 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:15,062 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. after waiting 0 ms 2024-11-24T15:28:15,062 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:15,065 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1732462090214.5f473e62c97758cd46c46a7e69adefcd. 2024-11-24T15:28:15,065 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5f473e62c97758cd46c46a7e69adefcd: Waiting for close lock at 1732462095062Disabling compacts and flushes for region at 1732462095062Disabling writes for close at 1732462095062Writing region close event to WAL at 1732462095065 (+3 ms)Closed at 1732462095065 2024-11-24T15:28:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741934_1114 (size=175) 2024-11-24T15:28:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741934_1114 (size=175) 2024-11-24T15:28:15,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741934_1114 (size=175) 2024-11-24T15:28:15,069 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-24T15:28:15,069 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1732462094850) 2024-11-24T15:28:15,084 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 441) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44579 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_590994328_22 at /127.0.0.1:42072 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_590994328_22 at /127.0.0.1:49864 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44579 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_590994328_22 at /127.0.0.1:55600 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=156 (was 169), ProcessCount=11 (was 11), AvailableMemoryMB=6269 (was 6300) 2024-11-24T15:28:15,084 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-24T15:28:15,084 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-24T15:28:15,084 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T15:28:15,084 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T15:28:15,084 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,085 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,085 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-24T15:28:15,085 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-24T15:28:15,085 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1088696683, stopped=false 2024-11-24T15:28:15,085 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c99348da3e93,40561,1732462054824 2024-11-24T15:28:15,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T15:28:15,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T15:28:15,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-24T15:28:15,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:28:15,087 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:28:15,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:28:15,088 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T15:28:15,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:28:15,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:28:15,089 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-24T15:28:15,089 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-24T15:28:15,089 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T15:28:15,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,089 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c99348da3e93,38975,1732462055523' ***** 2024-11-24T15:28:15,089 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T15:28:15,089 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c99348da3e93,33951,1732462055634' ***** 2024-11-24T15:28:15,089 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-24T15:28:15,090 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-24T15:28:15,090 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(3091): Received CLOSE for bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(959): stopping server c99348da3e93,33951,1732462055634 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(959): stopping server c99348da3e93,38975,1732462055523 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c99348da3e93:38975. 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c99348da3e93:33951. 2024-11-24T15:28:15,090 DEBUG [RS:0;c99348da3e93:38975 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T15:28:15,090 DEBUG [RS:1;c99348da3e93:33951 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-24T15:28:15,090 DEBUG [RS:0;c99348da3e93:38975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,090 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing bfa80e75ea1b7758b9c599d2ce0ef545, disabling compactions & flushes 2024-11-24T15:28:15,090 DEBUG [RS:1;c99348da3e93:33951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,090 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:28:15,090 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T15:28:15,090 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T15:28:15,090 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1325): Online Regions={bfa80e75ea1b7758b9c599d2ce0ef545=testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545.} 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T15:28:15,090 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. after waiting 0 ms 2024-11-24T15:28:15,090 DEBUG [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1351): Waiting on bfa80e75ea1b7758b9c599d2ce0ef545 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T15:28:15,090 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:28:15,090 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-24T15:28:15,091 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-24T15:28:15,091 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-24T15:28:15,091 DEBUG [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-24T15:28:15,091 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-24T15:28:15,091 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-24T15:28:15,091 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-24T15:28:15,091 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-24T15:28:15,091 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-24T15:28:15,091 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-24T15:28:15,095 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/default/testReplayEditsAfterRegionMovedWithMultiCF/bfa80e75ea1b7758b9c599d2ce0ef545/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-24T15:28:15,096 INFO [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:28:15,096 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for bfa80e75ea1b7758b9c599d2ce0ef545: Waiting for close lock at 1732462095090Running coprocessor pre-close hooks at 1732462095090Disabling compacts and flushes for region at 1732462095090Disabling writes for close at 1732462095090Writing region close event to WAL at 1732462095092 (+2 ms)Running coprocessor post-close hooks at 1732462095096 (+4 ms)Closed at 1732462095096 2024-11-24T15:28:15,096 DEBUG [RS_CLOSE_REGION-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545. 2024-11-24T15:28:15,113 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/info/f7e9eb78028c4d3188f0b4f485f4f54f is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1732462071030.bfa80e75ea1b7758b9c599d2ce0ef545./info:regioninfo/1732462074103/Put/seqid=0 2024-11-24T15:28:15,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741938_1118 (size=8243) 2024-11-24T15:28:15,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741938_1118 (size=8243) 2024-11-24T15:28:15,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741938_1118 (size=8243) 2024-11-24T15:28:15,120 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/info/f7e9eb78028c4d3188f0b4f485f4f54f 2024-11-24T15:28:15,140 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/ns/73f16f7c86d54911b7cf0f29174140d2 is 43, key is default/ns:d/1732462057676/Put/seqid=0 2024-11-24T15:28:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741939_1119 (size=5153) 2024-11-24T15:28:15,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741939_1119 (size=5153) 2024-11-24T15:28:15,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741939_1119 (size=5153) 2024-11-24T15:28:15,147 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/ns/73f16f7c86d54911b7cf0f29174140d2 2024-11-24T15:28:15,167 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/table/772eae5fbdc347adabb709893320a819 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1732462071442/Put/seqid=0 2024-11-24T15:28:15,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741940_1120 (size=5431) 2024-11-24T15:28:15,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741940_1120 (size=5431) 2024-11-24T15:28:15,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741940_1120 (size=5431) 2024-11-24T15:28:15,174 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/table/772eae5fbdc347adabb709893320a819 2024-11-24T15:28:15,174 INFO [regionserver/c99348da3e93:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:28:15,174 INFO [regionserver/c99348da3e93:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:28:15,179 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/info/f7e9eb78028c4d3188f0b4f485f4f54f as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/info/f7e9eb78028c4d3188f0b4f485f4f54f 2024-11-24T15:28:15,184 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/info/f7e9eb78028c4d3188f0b4f485f4f54f, entries=18, sequenceid=21, filesize=8.0 K 2024-11-24T15:28:15,185 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/ns/73f16f7c86d54911b7cf0f29174140d2 as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/ns/73f16f7c86d54911b7cf0f29174140d2 2024-11-24T15:28:15,190 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/ns/73f16f7c86d54911b7cf0f29174140d2, entries=2, sequenceid=21, filesize=5.0 K 2024-11-24T15:28:15,191 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/.tmp/table/772eae5fbdc347adabb709893320a819 as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/table/772eae5fbdc347adabb709893320a819 2024-11-24T15:28:15,195 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/table/772eae5fbdc347adabb709893320a819, entries=2, sequenceid=21, filesize=5.3 K 2024-11-24T15:28:15,196 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false 2024-11-24T15:28:15,200 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-24T15:28:15,200 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T15:28:15,200 INFO [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-24T15:28:15,201 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732462095091Running coprocessor pre-close hooks at 1732462095091Disabling compacts and flushes for region at 1732462095091Disabling writes for close at 1732462095091Obtaining lock to block concurrent updates at 1732462095091Preparing flush snapshotting stores in 1588230740 at 1732462095091Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1732462095092 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732462095092Flushing 1588230740/info: creating writer at 1732462095093 (+1 ms)Flushing 1588230740/info: appending metadata at 1732462095112 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732462095112Flushing 1588230740/ns: creating writer at 1732462095125 (+13 ms)Flushing 1588230740/ns: appending metadata at 1732462095140 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732462095140Flushing 1588230740/table: creating writer at 1732462095153 (+13 ms)Flushing 1588230740/table: appending metadata at 1732462095167 (+14 ms)Flushing 1588230740/table: closing flushed file at 1732462095167Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@35ce2213: reopening flushed file at 1732462095179 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19729ad3: reopening flushed file at 1732462095185 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ab29239: reopening flushed file at 1732462095190 (+5 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 105ms, sequenceid=21, compaction requested=false at 1732462095196 (+6 ms)Writing region close event to WAL at 1732462095197 (+1 ms)Running coprocessor post-close hooks at 1732462095200 (+3 ms)Closed at 1732462095200 2024-11-24T15:28:15,201 DEBUG [RS_CLOSE_META-regionserver/c99348da3e93:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-24T15:28:15,243 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-24T15:28:15,243 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-24T15:28:15,244 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-24T15:28:15,244 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-24T15:28:15,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-24T15:28:15,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-24T15:28:15,245 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-24T15:28:15,291 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(976): stopping server c99348da3e93,38975,1732462055523; all regions closed. 2024-11-24T15:28:15,291 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(976): stopping server c99348da3e93,33951,1732462055634; all regions closed. 2024-11-24T15:28:15,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741833_1009 (size=3621) 2024-11-24T15:28:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741833_1009 (size=3621) 2024-11-24T15:28:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741836_1012 (size=9487) 2024-11-24T15:28:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741833_1009 (size=3621) 2024-11-24T15:28:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741836_1012 (size=9487) 2024-11-24T15:28:15,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741836_1012 (size=9487) 2024-11-24T15:28:15,297 DEBUG [RS:0;c99348da3e93:38975 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c99348da3e93%2C38975%2C1732462055523:(num 1732462056994) 2024-11-24T15:28:15,297 DEBUG [RS:0;c99348da3e93:38975 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:28:15,297 DEBUG [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs 2024-11-24T15:28:15,297 INFO [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c99348da3e93%2C33951%2C1732462055634.meta:.meta(num 1732462057453) 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] hbase.ChoreService(370): Chore service for: regionserver/c99348da3e93:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-24T15:28:15,297 INFO [regionserver/c99348da3e93:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T15:28:15,297 INFO [RS:0;c99348da3e93:38975 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:38975 2024-11-24T15:28:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741835_1011 (size=175) 2024-11-24T15:28:15,300 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/WALs/c99348da3e93,33951,1732462055634/c99348da3e93%2C33951%2C1732462055634.1732462056995 not finished, retry = 0 2024-11-24T15:28:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741835_1011 (size=175) 2024-11-24T15:28:15,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741835_1011 (size=175) 2024-11-24T15:28:15,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c99348da3e93,38975,1732462055523 2024-11-24T15:28:15,300 INFO [RS:0;c99348da3e93:38975 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T15:28:15,300 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T15:28:15,302 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c99348da3e93,38975,1732462055523] 2024-11-24T15:28:15,304 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c99348da3e93,38975,1732462055523 already deleted, retry=false 2024-11-24T15:28:15,304 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c99348da3e93,38975,1732462055523 expired; onlineServers=1 2024-11-24T15:28:15,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,402 INFO [RS:0;c99348da3e93:38975 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T15:28:15,402 INFO [RS:0;c99348da3e93:38975 {}] regionserver.HRegionServer(1031): Exiting; stopping=c99348da3e93,38975,1732462055523; zookeeper connection closed. 2024-11-24T15:28:15,402 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:38975-0x1014a96f51f0001, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,402 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@28719e47 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@28719e47 2024-11-24T15:28:15,403 DEBUG [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/oldWALs 2024-11-24T15:28:15,403 INFO [RS:1;c99348da3e93:33951 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c99348da3e93%2C33951%2C1732462055634:(num 1732462056995) 2024-11-24T15:28:15,403 DEBUG [RS:1;c99348da3e93:33951 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-24T15:28:15,403 INFO [RS:1;c99348da3e93:33951 {}] regionserver.LeaseManager(133): Closed leases 2024-11-24T15:28:15,403 INFO [RS:1;c99348da3e93:33951 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T15:28:15,403 INFO [RS:1;c99348da3e93:33951 {}] hbase.ChoreService(370): Chore service for: regionserver/c99348da3e93:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-24T15:28:15,403 INFO [RS:1;c99348da3e93:33951 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T15:28:15,403 INFO [regionserver/c99348da3e93:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T15:28:15,404 INFO [RS:1;c99348da3e93:33951 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33951 2024-11-24T15:28:15,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-24T15:28:15,406 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c99348da3e93,33951,1732462055634 2024-11-24T15:28:15,406 INFO [RS:1;c99348da3e93:33951 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T15:28:15,407 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c99348da3e93,33951,1732462055634] 2024-11-24T15:28:15,408 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c99348da3e93,33951,1732462055634 already deleted, retry=false 2024-11-24T15:28:15,408 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c99348da3e93,33951,1732462055634 expired; onlineServers=0 2024-11-24T15:28:15,408 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c99348da3e93,40561,1732462054824' ***** 2024-11-24T15:28:15,408 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-24T15:28:15,409 DEBUG [M:0;c99348da3e93:40561 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-24T15:28:15,409 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-24T15:28:15,409 DEBUG [M:0;c99348da3e93:40561 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-24T15:28:15,409 DEBUG [master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.large.0-1732462056686 {}] cleaner.HFileCleaner(306): Exit Thread[master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.large.0-1732462056686,5,FailOnTimeoutGroup] 2024-11-24T15:28:15,409 DEBUG [master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.small.0-1732462056688 {}] cleaner.HFileCleaner(306): Exit Thread[master/c99348da3e93:0:becomeActiveMaster-HFileCleaner.small.0-1732462056688,5,FailOnTimeoutGroup] 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] hbase.ChoreService(370): Chore service for: master/c99348da3e93:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-24T15:28:15,409 DEBUG [M:0;c99348da3e93:40561 {}] master.HMaster(1795): Stopping service threads 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-24T15:28:15,409 INFO [M:0;c99348da3e93:40561 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-24T15:28:15,410 INFO [M:0;c99348da3e93:40561 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-24T15:28:15,410 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-24T15:28:15,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-24T15:28:15,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-24T15:28:15,412 DEBUG [M:0;c99348da3e93:40561 {}] zookeeper.ZKUtil(347): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-24T15:28:15,412 WARN [M:0;c99348da3e93:40561 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-24T15:28:15,413 INFO [M:0;c99348da3e93:40561 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/.lastflushedseqids 2024-11-24T15:28:15,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741941_1121 (size=138) 2024-11-24T15:28:15,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741941_1121 (size=138) 2024-11-24T15:28:15,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741941_1121 (size=138) 2024-11-24T15:28:15,425 INFO [M:0;c99348da3e93:40561 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-24T15:28:15,425 INFO [M:0;c99348da3e93:40561 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-24T15:28:15,425 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-24T15:28:15,425 INFO [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:28:15,426 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:28:15,426 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-24T15:28:15,426 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:28:15,426 INFO [M:0;c99348da3e93:40561 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-11-24T15:28:15,441 DEBUG [M:0;c99348da3e93:40561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dc2a67f3a7b43118ee58bcbad334b0f is 82, key is hbase:meta,,1/info:regioninfo/1732462057567/Put/seqid=0 2024-11-24T15:28:15,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741942_1122 (size=5672) 2024-11-24T15:28:15,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741942_1122 (size=5672) 2024-11-24T15:28:15,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741942_1122 (size=5672) 2024-11-24T15:28:15,448 INFO [M:0;c99348da3e93:40561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dc2a67f3a7b43118ee58bcbad334b0f 2024-11-24T15:28:15,466 DEBUG [M:0;c99348da3e93:40561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/785231c949ac497f8d7a85dd7fe2b8aa is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1732462071448/Put/seqid=0 2024-11-24T15:28:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741943_1123 (size=7756) 2024-11-24T15:28:15,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741943_1123 (size=7756) 2024-11-24T15:28:15,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741943_1123 (size=7756) 2024-11-24T15:28:15,473 INFO [M:0;c99348da3e93:40561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/785231c949ac497f8d7a85dd7fe2b8aa 2024-11-24T15:28:15,477 INFO [M:0;c99348da3e93:40561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 785231c949ac497f8d7a85dd7fe2b8aa 2024-11-24T15:28:15,491 DEBUG [M:0;c99348da3e93:40561 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e685801dfcc458a82ddd25068211dca is 69, key is c99348da3e93,33951,1732462055634/rs:state/1732462056736/Put/seqid=0 2024-11-24T15:28:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741944_1124 (size=5440) 2024-11-24T15:28:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741944_1124 (size=5440) 2024-11-24T15:28:15,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741944_1124 (size=5440) 2024-11-24T15:28:15,498 INFO [M:0;c99348da3e93:40561 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e685801dfcc458a82ddd25068211dca 2024-11-24T15:28:15,502 INFO [M:0;c99348da3e93:40561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e685801dfcc458a82ddd25068211dca 2024-11-24T15:28:15,502 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/1dc2a67f3a7b43118ee58bcbad334b0f as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1dc2a67f3a7b43118ee58bcbad334b0f 2024-11-24T15:28:15,506 INFO [M:0;c99348da3e93:40561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/1dc2a67f3a7b43118ee58bcbad334b0f, entries=8, sequenceid=168, filesize=5.5 K 2024-11-24T15:28:15,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,507 INFO [RS:1;c99348da3e93:33951 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T15:28:15,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33951-0x1014a96f51f0002, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,507 INFO [RS:1;c99348da3e93:33951 {}] regionserver.HRegionServer(1031): Exiting; stopping=c99348da3e93,33951,1732462055634; zookeeper connection closed. 2024-11-24T15:28:15,507 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/785231c949ac497f8d7a85dd7fe2b8aa as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/785231c949ac497f8d7a85dd7fe2b8aa 2024-11-24T15:28:15,507 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@44da37a6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@44da37a6 2024-11-24T15:28:15,508 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-24T15:28:15,512 INFO [M:0;c99348da3e93:40561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 785231c949ac497f8d7a85dd7fe2b8aa 2024-11-24T15:28:15,512 INFO [M:0;c99348da3e93:40561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/785231c949ac497f8d7a85dd7fe2b8aa, entries=17, sequenceid=168, filesize=7.6 K 2024-11-24T15:28:15,513 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/0e685801dfcc458a82ddd25068211dca as hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0e685801dfcc458a82ddd25068211dca 2024-11-24T15:28:15,517 INFO [M:0;c99348da3e93:40561 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0e685801dfcc458a82ddd25068211dca 2024-11-24T15:28:15,517 INFO [M:0;c99348da3e93:40561 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44579/user/jenkins/test-data/00ac5bc2-b717-5ea1-5230-73c8f73af08f/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/0e685801dfcc458a82ddd25068211dca, entries=3, sequenceid=168, filesize=5.3 K 2024-11-24T15:28:15,518 INFO [M:0;c99348da3e93:40561 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false 2024-11-24T15:28:15,520 INFO [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-24T15:28:15,520 DEBUG [M:0;c99348da3e93:40561 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732462095425Disabling compacts and flushes for region at 1732462095425Disabling writes for close at 1732462095426 (+1 ms)Obtaining lock to block concurrent updates at 1732462095426Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732462095426Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1732462095426Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732462095427 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732462095427Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732462095441 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732462095441Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732462095452 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732462095466 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732462095466Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732462095477 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732462095491 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732462095491Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f35ac6: reopening flushed file at 1732462095502 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16cb175e: reopening flushed file at 1732462095506 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@353eb497: reopening flushed file at 1732462095513 (+7 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 92ms, sequenceid=168, compaction requested=false at 1732462095518 (+5 ms)Writing region close event to WAL at 1732462095519 (+1 ms)Closed at 1732462095519 2024-11-24T15:28:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741830_1006 (size=83618) 2024-11-24T15:28:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41867 is added to blk_1073741830_1006 (size=83618) 2024-11-24T15:28:15,522 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:39259 is added to blk_1073741830_1006 (size=83618) 2024-11-24T15:28:15,523 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-24T15:28:15,523 INFO [M:0;c99348da3e93:40561 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-24T15:28:15,523 INFO [M:0;c99348da3e93:40561 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40561 2024-11-24T15:28:15,523 INFO [M:0;c99348da3e93:40561 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-24T15:28:15,625 INFO [M:0;c99348da3e93:40561 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-24T15:28:15,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40561-0x1014a96f51f0000, quorum=127.0.0.1:64743, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-24T15:28:15,632 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462075002 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462075002 (inode 16655) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1732462074472/wal.1732462075002 (inode 16655) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T15:28:15,634 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1732462066471/wal.1732462066607 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:15,634 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1732462090213/wal.1732462090534 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:15,636 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462084546 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462084546 (inode 16678) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1732462075196/wal.1732462084546 (inode 16678) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T15:28:15,637 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1732462059258/wal.1732462059361 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:15,639 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462085450 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462085450 (inode 16704) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1732462084809/wal.1732462085450 (inode 16704) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T15:28:15,640 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1732462085643/wal.1732462085732 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-24T15:28:15,643 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal.1732462059057 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal.1732462059057 (inode 16485) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1732462058645/wal.1732462059057 (inode 16485) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T15:28:15,645 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal.1732462058363 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal.1732462058363 (inode 16462) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1732462057916/wal.1732462058363 (inode 16462) Holder DFSClient_NONMAPREDUCE_460530494_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-24T15:28:15,650 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:28:15,652 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T15:28:15,652 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T15:28:15,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T15:28:15,652 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,STOPPED} 2024-11-24T15:28:15,655 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T15:28:15,655 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T15:28:15,655 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T15:28:15,655 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-532991956-172.17.0.2-1732462051681 (Datanode Uuid a852540f-7401-4edc-a24f-ae014780e5b6) service to localhost/127.0.0.1:44579 2024-11-24T15:28:15,657 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data5/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,657 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data6/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,657 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T15:28:15,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:28:15,660 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T15:28:15,660 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T15:28:15,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T15:28:15,660 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,STOPPED} 2024-11-24T15:28:15,661 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T15:28:15,661 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T15:28:15,661 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T15:28:15,661 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-532991956-172.17.0.2-1732462051681 (Datanode Uuid f20b3dfe-433d-47b8-830a-34624f723627) service to localhost/127.0.0.1:44579 2024-11-24T15:28:15,662 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data3/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,662 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data4/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,662 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T15:28:15,664 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-24T15:28:15,664 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T15:28:15,665 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T15:28:15,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T15:28:15,665 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,STOPPED} 2024-11-24T15:28:15,666 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-24T15:28:15,666 WARN [BP-532991956-172.17.0.2-1732462051681 heartbeating to localhost/127.0.0.1:44579 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-532991956-172.17.0.2-1732462051681 (Datanode Uuid 3caae417-876c-4922-be6c-aa2097011217) service to localhost/127.0.0.1:44579 2024-11-24T15:28:15,666 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-24T15:28:15,666 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-24T15:28:15,666 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data1/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,667 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/cluster_3df2de49-26d9-c094-22aa-e5a5e32f540f/data/data2/current/BP-532991956-172.17.0.2-1732462051681 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-24T15:28:15,667 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-24T15:28:15,674 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-24T15:28:15,674 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-24T15:28:15,674 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-24T15:28:15,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-24T15:28:15,675 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/6c4d0879-325e-10b2-1aed-71d01a738857/hadoop.log.dir/,STOPPED} 2024-11-24T15:28:15,683 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-24T15:28:15,728 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down