2024-11-13 18:25:22,261 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 18:25:22,278 main DEBUG Took 0.014183 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 18:25:22,278 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 18:25:22,279 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 18:25:22,280 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 18:25:22,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,291 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 18:25:22,307 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,309 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,310 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,312 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,313 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,315 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,317 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,317 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,318 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,319 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,319 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,320 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,320 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,321 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,321 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 18:25:22,322 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,322 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 18:25:22,324 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 18:25:22,326 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 18:25:22,328 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 18:25:22,329 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 18:25:22,330 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 18:25:22,331 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 18:25:22,343 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 18:25:22,346 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 18:25:22,348 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 18:25:22,349 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 18:25:22,349 main DEBUG createAppenders(={Console}) 2024-11-13 18:25:22,350 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-13 18:25:22,350 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 18:25:22,351 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-13 18:25:22,351 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 18:25:22,352 main DEBUG OutputStream closed 2024-11-13 18:25:22,352 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 18:25:22,353 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 18:25:22,353 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-13 18:25:22,433 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 18:25:22,435 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 18:25:22,436 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 18:25:22,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 18:25:22,437 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 18:25:22,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 18:25:22,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 18:25:22,438 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 18:25:22,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 18:25:22,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 18:25:22,439 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 18:25:22,440 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 18:25:22,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 18:25:22,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 18:25:22,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 18:25:22,441 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 18:25:22,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 18:25:22,442 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 18:25:22,445 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 18:25:22,445 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-13 18:25:22,446 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 18:25:22,447 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-13T18:25:22,772 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2 2024-11-13 18:25:22,776 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 18:25:22,776 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T18:25:22,788 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T18:25:22,820 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T18:25:22,875 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T18:25:22,876 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T18:25:22,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T18:25:22,906 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88, deleteOnExit=true 2024-11-13T18:25:22,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T18:25:22,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/test.cache.data in system properties and HBase conf 2024-11-13T18:25:22,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T18:25:22,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir in system properties and HBase conf 2024-11-13T18:25:22,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T18:25:22,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T18:25:22,910 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T18:25:22,992 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T18:25:23,108 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T18:25:23,113 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:25:23,114 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T18:25:23,115 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T18:25:23,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:25:23,116 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T18:25:23,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T18:25:23,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T18:25:23,118 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:25:23,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T18:25:23,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/nfs.dump.dir in system properties and HBase conf 2024-11-13T18:25:23,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/java.io.tmpdir in system properties and HBase conf 2024-11-13T18:25:23,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T18:25:23,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T18:25:23,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T18:25:24,126 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T18:25:24,217 INFO [Time-limited test {}] log.Log(170): Logging initialized @2916ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T18:25:24,311 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:25:24,406 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:25:24,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:25:24,449 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:25:24,451 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T18:25:24,468 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:25:24,472 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:25:24,473 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:25:24,716 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/java.io.tmpdir/jetty-localhost-39819-hadoop-hdfs-3_4_1-tests_jar-_-any-10128772026679540091/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:25:24,725 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:39819} 2024-11-13T18:25:24,726 INFO [Time-limited test {}] server.Server(415): Started @3425ms 2024-11-13T18:25:25,179 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:25:25,189 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:25:25,191 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:25:25,191 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:25:25,192 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T18:25:25,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:25:25,194 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:25:25,329 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/java.io.tmpdir/jetty-localhost-34519-hadoop-hdfs-3_4_1-tests_jar-_-any-11709616023818079900/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:25:25,330 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:34519} 2024-11-13T18:25:25,331 INFO [Time-limited test {}] server.Server(415): Started @4030ms 2024-11-13T18:25:25,389 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:25:25,577 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:25:25,583 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:25:25,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:25:25,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:25:25,592 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:25:25,593 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:25:25,594 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:25:25,729 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/java.io.tmpdir/jetty-localhost-41937-hadoop-hdfs-3_4_1-tests_jar-_-any-14922635507486174836/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:25:25,730 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:41937} 2024-11-13T18:25:25,730 INFO [Time-limited test {}] server.Server(415): Started @4429ms 2024-11-13T18:25:25,732 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:25:25,783 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T18:25:25,792 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T18:25:25,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T18:25:25,794 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T18:25:25,795 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T18:25:25,797 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,AVAILABLE} 2024-11-13T18:25:25,798 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T18:25:25,910 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data3/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:25,917 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data1/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:25,917 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data4/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:25,918 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data2/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:25,950 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/java.io.tmpdir/jetty-localhost-44577-hadoop-hdfs-3_4_1-tests_jar-_-any-5689180121310579863/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:25:25,960 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:44577} 2024-11-13T18:25:25,961 INFO [Time-limited test {}] server.Server(415): Started @4660ms 2024-11-13T18:25:25,963 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T18:25:25,988 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:25:25,988 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:25:26,084 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7f6592fd9c07ef1 with lease ID 0x5593a3525a26c3b4: Processing first storage report for DS-9768edff-b9ba-44c8-ab76-6ac67e41de83 from datanode DatanodeRegistration(127.0.0.1:33523, datanodeUuid=bf437f33-3b70-4d83-a1a4-46e973f1f052, infoPort=36085, infoSecurePort=0, ipcPort=36759, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,085 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7f6592fd9c07ef1 with lease ID 0x5593a3525a26c3b4: from storage DS-9768edff-b9ba-44c8-ab76-6ac67e41de83 node DatanodeRegistration(127.0.0.1:33523, datanodeUuid=bf437f33-3b70-4d83-a1a4-46e973f1f052, infoPort=36085, infoSecurePort=0, ipcPort=36759, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x677b63a2f6a4efd8 with lease ID 0x5593a3525a26c3b5: Processing first storage report for DS-113ec9e2-4543-4828-b945-f3ecb16f5df1 from datanode DatanodeRegistration(127.0.0.1:46817, datanodeUuid=1b19a530-625c-4e79-b632-9290c2b4e239, infoPort=44725, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,086 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677b63a2f6a4efd8 with lease ID 0x5593a3525a26c3b5: from storage DS-113ec9e2-4543-4828-b945-f3ecb16f5df1 node DatanodeRegistration(127.0.0.1:46817, datanodeUuid=1b19a530-625c-4e79-b632-9290c2b4e239, infoPort=44725, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,087 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd7f6592fd9c07ef1 with lease ID 0x5593a3525a26c3b4: Processing first storage report for DS-0232cc51-7680-4bc2-ae40-61abda8026ae from datanode DatanodeRegistration(127.0.0.1:33523, datanodeUuid=bf437f33-3b70-4d83-a1a4-46e973f1f052, infoPort=36085, infoSecurePort=0, ipcPort=36759, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd7f6592fd9c07ef1 with lease ID 0x5593a3525a26c3b4: from storage DS-0232cc51-7680-4bc2-ae40-61abda8026ae node DatanodeRegistration(127.0.0.1:33523, datanodeUuid=bf437f33-3b70-4d83-a1a4-46e973f1f052, infoPort=36085, infoSecurePort=0, ipcPort=36759, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,087 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x677b63a2f6a4efd8 with lease ID 0x5593a3525a26c3b5: Processing first storage report for DS-a2e40dbc-42e9-4305-a687-78d7f22f6fc6 from datanode DatanodeRegistration(127.0.0.1:46817, datanodeUuid=1b19a530-625c-4e79-b632-9290c2b4e239, infoPort=44725, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,088 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x677b63a2f6a4efd8 with lease ID 0x5593a3525a26c3b5: from storage DS-a2e40dbc-42e9-4305-a687-78d7f22f6fc6 node DatanodeRegistration(127.0.0.1:46817, datanodeUuid=1b19a530-625c-4e79-b632-9290c2b4e239, infoPort=44725, infoSecurePort=0, ipcPort=41289, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,174 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data6/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:26,174 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data5/current/BP-1832393325-172.17.0.3-1731522323869/current, will proceed with Du for space computation calculation, 2024-11-13T18:25:26,219 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T18:25:26,225 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8876d3cd12643a7 with lease ID 0x5593a3525a26c3b6: Processing first storage report for DS-36951baa-cfb8-4898-850b-e4f621da6010 from datanode DatanodeRegistration(127.0.0.1:41041, datanodeUuid=af374aac-32d3-409a-b3c5-fa6d863690b8, infoPort=41527, infoSecurePort=0, ipcPort=37645, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,225 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8876d3cd12643a7 with lease ID 0x5593a3525a26c3b6: from storage DS-36951baa-cfb8-4898-850b-e4f621da6010 node DatanodeRegistration(127.0.0.1:41041, datanodeUuid=af374aac-32d3-409a-b3c5-fa6d863690b8, infoPort=41527, infoSecurePort=0, ipcPort=37645, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,226 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xb8876d3cd12643a7 with lease ID 0x5593a3525a26c3b6: Processing first storage report for DS-2bd8e5f9-9671-4911-878d-af90d552b89e from datanode DatanodeRegistration(127.0.0.1:41041, datanodeUuid=af374aac-32d3-409a-b3c5-fa6d863690b8, infoPort=41527, infoSecurePort=0, ipcPort=37645, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869) 2024-11-13T18:25:26,226 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xb8876d3cd12643a7 with lease ID 0x5593a3525a26c3b6: from storage DS-2bd8e5f9-9671-4911-878d-af90d552b89e node DatanodeRegistration(127.0.0.1:41041, datanodeUuid=af374aac-32d3-409a-b3c5-fa6d863690b8, infoPort=41527, infoSecurePort=0, ipcPort=37645, storageInfo=lv=-57;cid=testClusterID;nsid=596043917;c=1731522323869), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T18:25:26,469 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2 2024-11-13T18:25:26,586 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/zookeeper_0, clientPort=54860, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T18:25:26,597 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=54860 2024-11-13T18:25:26,609 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:26,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:26,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:25:26,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:25:26,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741825_1001 (size=7) 2024-11-13T18:25:27,267 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 with version=8 2024-11-13T18:25:27,267 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/hbase-staging 2024-11-13T18:25:27,647 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:25:27,659 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:27,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:27,665 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:25:27,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:27,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:25:27,814 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T18:25:27,883 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T18:25:27,895 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T18:25:27,899 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:25:27,933 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7694 (auto-detected) 2024-11-13T18:25:27,935 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T18:25:27,955 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42825 2024-11-13T18:25:27,977 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:42825 connecting to ZooKeeper ensemble=127.0.0.1:54860 2024-11-13T18:25:28,011 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:428250x0, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:25:28,015 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:42825-0x1003a6402a40000 connected 2024-11-13T18:25:28,079 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,083 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,110 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:25:28,115 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4, hbase.cluster.distributed=false 2024-11-13T18:25:28,153 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:25:28,169 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-13T18:25:28,172 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42825 2024-11-13T18:25:28,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42825 2024-11-13T18:25:28,192 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-13T18:25:28,200 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42825 2024-11-13T18:25:28,364 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:25:28,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,367 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:25:28,367 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,368 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:25:28,371 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:25:28,374 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:25:28,375 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:33531 2024-11-13T18:25:28,378 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:33531 connecting to ZooKeeper ensemble=127.0.0.1:54860 2024-11-13T18:25:28,380 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,385 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,392 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:335310x0, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:25:28,393 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:33531-0x1003a6402a40001 connected 2024-11-13T18:25:28,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:25:28,398 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:25:28,406 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:25:28,408 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:25:28,413 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:25:28,414 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33531 2024-11-13T18:25:28,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33531 2024-11-13T18:25:28,415 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33531 2024-11-13T18:25:28,416 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33531 2024-11-13T18:25:28,418 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33531 2024-11-13T18:25:28,436 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:25:28,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,436 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,437 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:25:28,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,437 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:25:28,437 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:25:28,438 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:25:28,438 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:44035 2024-11-13T18:25:28,440 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:44035 connecting to ZooKeeper ensemble=127.0.0.1:54860 2024-11-13T18:25:28,441 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,446 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,455 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:440350x0, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:25:28,456 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:44035-0x1003a6402a40002 connected 2024-11-13T18:25:28,456 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:25:28,457 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:25:28,458 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:25:28,459 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:25:28,461 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:25:28,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-13T18:25:28,462 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=44035 2024-11-13T18:25:28,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=44035 2024-11-13T18:25:28,463 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-13T18:25:28,464 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=44035 2024-11-13T18:25:28,490 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fda96cf025dc:0 server-side Connection retries=45 2024-11-13T18:25:28,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,490 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T18:25:28,490 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T18:25:28,491 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T18:25:28,491 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T18:25:28,491 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T18:25:28,492 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41739 2024-11-13T18:25:28,494 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41739 connecting to ZooKeeper ensemble=127.0.0.1:54860 2024-11-13T18:25:28,496 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,499 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417390x0, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T18:25:28,507 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417390x0, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:25:28,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41739-0x1003a6402a40003 connected 2024-11-13T18:25:28,508 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T18:25:28,510 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T18:25:28,511 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T18:25:28,513 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T18:25:28,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41739 2024-11-13T18:25:28,516 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41739 2024-11-13T18:25:28,520 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41739 2024-11-13T18:25:28,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41739 2024-11-13T18:25:28,521 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41739 2024-11-13T18:25:28,538 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fda96cf025dc:42825 2024-11-13T18:25:28,539 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fda96cf025dc,42825,1731522327393 2024-11-13T18:25:28,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,549 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fda96cf025dc,42825,1731522327393 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,574 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,576 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T18:25:28,577 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fda96cf025dc,42825,1731522327393 from backup master directory 2024-11-13T18:25:28,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,580 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fda96cf025dc,42825,1731522327393 2024-11-13T18:25:28,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,583 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:25:28,584 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fda96cf025dc,42825,1731522327393 2024-11-13T18:25:28,584 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T18:25:28,587 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T18:25:28,588 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T18:25:28,662 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/hbase.id] with ID: bd0e7932-69d1-4437-b0c5-be1e6f1fb465 2024-11-13T18:25:28,662 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/.tmp/hbase.id 2024-11-13T18:25:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:25:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:25:28,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741826_1002 (size=42) 2024-11-13T18:25:28,684 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/.tmp/hbase.id]:[hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/hbase.id] 2024-11-13T18:25:28,733 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T18:25:28,739 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T18:25:28,766 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 24ms. 2024-11-13T18:25:28,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,770 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:28,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:25:28,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:25:28,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741827_1003 (size=196) 2024-11-13T18:25:28,809 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:25:28,812 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T18:25:28,819 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:28,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:25:28,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:25:28,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741828_1004 (size=1189) 2024-11-13T18:25:28,892 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store 2024-11-13T18:25:28,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:25:28,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:25:28,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741829_1005 (size=34) 2024-11-13T18:25:28,932 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T18:25:28,936 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:28,938 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:25:28,938 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:25:28,939 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:25:28,941 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:25:28,941 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:25:28,941 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:25:28,943 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522328938Disabling compacts and flushes for region at 1731522328938Disabling writes for close at 1731522328941 (+3 ms)Writing region close event to WAL at 1731522328941Closed at 1731522328941 2024-11-13T18:25:28,945 WARN [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/.initializing 2024-11-13T18:25:28,946 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/WALs/fda96cf025dc,42825,1731522327393 2024-11-13T18:25:28,954 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:28,978 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C42825%2C1731522327393, suffix=, logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/WALs/fda96cf025dc,42825,1731522327393, archiveDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/oldWALs, maxLogs=10 2024-11-13T18:25:29,017 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/WALs/fda96cf025dc,42825,1731522327393/fda96cf025dc%2C42825%2C1731522327393.1731522328984, exclude list is [], retry=0 2024-11-13T18:25:29,045 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:29,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:29,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:29,053 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:29,057 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T18:25:29,120 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/WALs/fda96cf025dc,42825,1731522327393/fda96cf025dc%2C42825%2C1731522327393.1731522328984 2024-11-13T18:25:29,121 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:29,125 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:29,126 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:29,130 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,131 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,191 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,240 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T18:25:29,244 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,248 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:29,249 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,252 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T18:25:29,252 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,253 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:29,254 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,258 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T18:25:29,258 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:29,259 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,262 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T18:25:29,262 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,263 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:29,264 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,268 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,269 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,276 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,277 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,281 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:25:29,286 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T18:25:29,290 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:29,291 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73462946, jitterRate=0.0946832001209259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:25:29,297 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731522329150Initializing all the Stores at 1731522329158 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522329158Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522329161 (+3 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522329161Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522329161Cleaning up temporary data from old regions at 1731522329277 (+116 ms)Region opened successfully at 1731522329297 (+20 ms) 2024-11-13T18:25:29,299 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T18:25:29,338 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a755ac0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:25:29,373 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T18:25:29,386 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T18:25:29,386 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T18:25:29,389 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T18:25:29,391 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T18:25:29,397 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-13T18:25:29,397 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T18:25:29,426 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T18:25:29,435 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T18:25:29,438 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T18:25:29,440 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T18:25:29,442 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T18:25:29,445 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T18:25:29,447 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T18:25:29,450 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T18:25:29,452 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T18:25:29,453 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T18:25:29,455 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T18:25:29,473 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T18:25:29,475 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,479 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,482 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fda96cf025dc,42825,1731522327393, sessionid=0x1003a6402a40000, setting cluster-up flag (Was=false) 2024-11-13T18:25:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,496 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,503 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T18:25:29,505 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,42825,1731522327393 2024-11-13T18:25:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,512 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:29,518 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T18:25:29,519 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fda96cf025dc,42825,1731522327393 2024-11-13T18:25:29,526 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T18:25:29,625 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(746): ClusterId : bd0e7932-69d1-4437-b0c5-be1e6f1fb465 2024-11-13T18:25:29,625 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(746): ClusterId : bd0e7932-69d1-4437-b0c5-be1e6f1fb465 2024-11-13T18:25:29,625 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(746): ClusterId : bd0e7932-69d1-4437-b0c5-be1e6f1fb465 2024-11-13T18:25:29,625 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T18:25:29,628 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:25:29,628 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:25:29,628 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T18:25:29,636 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T18:25:29,641 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:25:29,641 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:25:29,641 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T18:25:29,641 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:25:29,641 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:25:29,641 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T18:25:29,644 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T18:25:29,646 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:25:29,646 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:25:29,646 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T18:25:29,647 DEBUG [RS:0;fda96cf025dc:33531 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43011139, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:25:29,647 DEBUG [RS:1;fda96cf025dc:44035 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@638829f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:25:29,647 DEBUG [RS:2;fda96cf025dc:41739 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b5b3870, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fda96cf025dc/172.17.0.3:0 2024-11-13T18:25:29,651 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fda96cf025dc,42825,1731522327393 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fda96cf025dc:0, corePoolSize=5, maxPoolSize=5 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fda96cf025dc:0, corePoolSize=10, maxPoolSize=10 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:25:29,659 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,671 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;fda96cf025dc:41739 2024-11-13T18:25:29,673 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;fda96cf025dc:44035 2024-11-13T18:25:29,674 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fda96cf025dc:33531 2024-11-13T18:25:29,676 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:25:29,676 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:25:29,676 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T18:25:29,676 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:25:29,676 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:25:29,676 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T18:25:29,676 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:25:29,676 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:25:29,676 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T18:25:29,679 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,42825,1731522327393 with port=44035, startcode=1731522328435 2024-11-13T18:25:29,679 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,42825,1731522327393 with port=41739, startcode=1731522328489 2024-11-13T18:25:29,679 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(2659): reportForDuty to master=fda96cf025dc,42825,1731522327393 with port=33531, startcode=1731522328314 2024-11-13T18:25:29,683 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731522359683 2024-11-13T18:25:29,683 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:25:29,684 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T18:25:29,685 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T18:25:29,686 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T18:25:29,691 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,692 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T18:25:29,692 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T18:25:29,692 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T18:25:29,693 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T18:25:29,693 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T18:25:29,694 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,700 DEBUG [RS:2;fda96cf025dc:41739 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:25:29,700 DEBUG [RS:0;fda96cf025dc:33531 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:25:29,700 DEBUG [RS:1;fda96cf025dc:44035 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T18:25:29,705 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T18:25:29,706 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T18:25:29,707 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T18:25:29,710 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T18:25:29,710 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T18:25:29,745 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522329741,5,FailOnTimeoutGroup] 2024-11-13T18:25:29,747 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522329745,5,FailOnTimeoutGroup] 2024-11-13T18:25:29,747 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,747 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T18:25:29,748 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:25:29,749 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:25:29,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741831_1007 (size=1321) 2024-11-13T18:25:29,761 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T18:25:29,762 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 2024-11-13T18:25:29,786 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36069, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:25:29,786 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38521, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:25:29,792 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56829, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T18:25:29,801 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,33531,1731522328314 2024-11-13T18:25:29,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:25:29,805 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,33531,1731522328314 2024-11-13T18:25:29,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:25:29,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741832_1008 (size=32) 2024-11-13T18:25:29,813 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:29,825 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,41739,1731522328489 2024-11-13T18:25:29,825 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:29,826 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:25:29,831 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 2024-11-13T18:25:29,832 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40367 2024-11-13T18:25:29,832 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:25:29,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fda96cf025dc,44035,1731522328435 2024-11-13T18:25:29,837 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=42825 {}] master.ServerManager(517): Registering regionserver=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:29,841 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 2024-11-13T18:25:29,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:25:29,841 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40367 2024-11-13T18:25:29,841 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:25:29,843 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:25:29,843 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,845 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:29,846 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:25:29,849 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:25:29,849 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,850 DEBUG [RS:0;fda96cf025dc:33531 {}] zookeeper.ZKUtil(111): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,33531,1731522328314 2024-11-13T18:25:29,850 WARN [RS:0;fda96cf025dc:33531 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:25:29,850 INFO [RS:0;fda96cf025dc:33531 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:29,850 DEBUG [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,33531,1731522328314 2024-11-13T18:25:29,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:29,852 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 2024-11-13T18:25:29,852 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40367 2024-11-13T18:25:29,852 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T18:25:29,852 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:25:29,853 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,33531,1731522328314] 2024-11-13T18:25:29,853 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,41739,1731522328489] 2024-11-13T18:25:29,856 DEBUG [RS:2;fda96cf025dc:41739 {}] zookeeper.ZKUtil(111): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,41739,1731522328489 2024-11-13T18:25:29,856 WARN [RS:2;fda96cf025dc:41739 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:25:29,856 INFO [RS:2;fda96cf025dc:41739 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:29,856 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489 2024-11-13T18:25:29,857 DEBUG [RS:1;fda96cf025dc:44035 {}] zookeeper.ZKUtil(111): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fda96cf025dc,44035,1731522328435 2024-11-13T18:25:29,857 WARN [RS:1;fda96cf025dc:44035 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T18:25:29,857 INFO [RS:1;fda96cf025dc:44035 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:29,857 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435 2024-11-13T18:25:29,858 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:25:29,858 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,859 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:29,860 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:25:29,862 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:25:29,864 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fda96cf025dc,44035,1731522328435] 2024-11-13T18:25:29,867 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:25:29,867 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:29,868 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:29,869 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:25:29,873 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740 2024-11-13T18:25:29,877 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740 2024-11-13T18:25:29,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:25:29,882 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:25:29,883 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:25:29,886 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:25:29,891 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:29,892 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70459624, jitterRate=0.04993021488189697}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:25:29,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731522329813Initializing all the Stores at 1731522329821 (+8 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522329821Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522329824 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522329824Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522329825 (+1 ms)Cleaning up temporary data from old regions at 1731522329882 (+57 ms)Region opened successfully at 1731522329896 (+14 ms) 2024-11-13T18:25:29,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:25:29,896 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:25:29,896 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:25:29,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:25:29,897 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:25:29,902 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:25:29,902 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522329896Disabling compacts and flushes for region at 1731522329896Disabling writes for close at 1731522329897 (+1 ms)Writing region close event to WAL at 1731522329901 (+4 ms)Closed at 1731522329902 (+1 ms) 2024-11-13T18:25:29,904 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:25:29,904 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:25:29,913 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:25:29,913 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T18:25:29,915 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T18:25:29,923 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T18:25:29,926 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:25:29,929 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:25:29,941 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:25:29,944 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T18:25:29,945 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T18:25:29,950 INFO [RS:0;fda96cf025dc:33531 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:25:29,951 INFO [RS:1;fda96cf025dc:44035 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:25:29,951 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,951 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,954 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:25:29,957 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:25:29,959 INFO [RS:2;fda96cf025dc:41739 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:25:29,959 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,960 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T18:25:29,963 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:25:29,964 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:25:29,965 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,965 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,965 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,965 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,965 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T18:25:29,965 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,965 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,965 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,965 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,965 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:25:29,966 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,966 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0, corePoolSize=2, maxPoolSize=2 2024-11-13T18:25:29,966 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:2;fda96cf025dc:41739 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,967 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fda96cf025dc:0, corePoolSize=1, maxPoolSize=1 2024-11-13T18:25:29,967 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,968 DEBUG [RS:0;fda96cf025dc:33531 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,968 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,968 DEBUG [RS:1;fda96cf025dc:44035 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fda96cf025dc:0, corePoolSize=3, maxPoolSize=3 2024-11-13T18:25:29,981 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,981 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,981 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,981 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,981 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,981 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,982 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41739,1731522328489-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:25:29,982 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,982 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,982 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,984 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,985 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,44035,1731522328435-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:25:29,988 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,989 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,989 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,989 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,989 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:29,989 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,33531,1731522328314-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:25:30,014 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:25:30,016 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:25:30,018 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,41739,1731522328489-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,018 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,44035,1731522328435-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,019 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,019 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,019 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.Replication(171): fda96cf025dc,44035,1731522328435 started 2024-11-13T18:25:30,019 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.Replication(171): fda96cf025dc,41739,1731522328489 started 2024-11-13T18:25:30,051 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,051 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,44035,1731522328435, RpcServer on fda96cf025dc/172.17.0.3:44035, sessionid=0x1003a6402a40002 2024-11-13T18:25:30,053 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:25:30,053 DEBUG [RS:1;fda96cf025dc:44035 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,44035,1731522328435 2024-11-13T18:25:30,054 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,44035,1731522328435' 2024-11-13T18:25:30,055 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:25:30,056 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,056 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,41739,1731522328489, RpcServer on fda96cf025dc/172.17.0.3:41739, sessionid=0x1003a6402a40003 2024-11-13T18:25:30,056 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T18:25:30,057 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:25:30,057 DEBUG [RS:2;fda96cf025dc:41739 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,41739,1731522328489 2024-11-13T18:25:30,057 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,41739,1731522328489' 2024-11-13T18:25:30,057 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,33531,1731522328314-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,057 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:25:30,057 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,059 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.Replication(171): fda96cf025dc,33531,1731522328314 started 2024-11-13T18:25:30,061 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:25:30,064 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:25:30,065 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:25:30,065 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:25:30,066 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:25:30,066 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:25:30,066 DEBUG [RS:2;fda96cf025dc:41739 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,41739,1731522328489 2024-11-13T18:25:30,067 DEBUG [RS:1;fda96cf025dc:44035 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,44035,1731522328435 2024-11-13T18:25:30,067 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,41739,1731522328489' 2024-11-13T18:25:30,067 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,44035,1731522328435' 2024-11-13T18:25:30,067 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:25:30,068 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:25:30,069 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:25:30,069 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:25:30,073 DEBUG [RS:2;fda96cf025dc:41739 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:25:30,073 INFO [RS:2;fda96cf025dc:41739 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:25:30,073 DEBUG [RS:1;fda96cf025dc:44035 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:25:30,073 INFO [RS:2;fda96cf025dc:41739 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:25:30,073 INFO [RS:1;fda96cf025dc:44035 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:25:30,073 INFO [RS:1;fda96cf025dc:44035 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:25:30,086 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,086 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1482): Serving as fda96cf025dc,33531,1731522328314, RpcServer on fda96cf025dc/172.17.0.3:33531, sessionid=0x1003a6402a40001 2024-11-13T18:25:30,087 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T18:25:30,087 DEBUG [RS:0;fda96cf025dc:33531 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fda96cf025dc,33531,1731522328314 2024-11-13T18:25:30,087 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,33531,1731522328314' 2024-11-13T18:25:30,087 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T18:25:30,089 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T18:25:30,093 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T18:25:30,093 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T18:25:30,093 DEBUG [RS:0;fda96cf025dc:33531 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fda96cf025dc,33531,1731522328314 2024-11-13T18:25:30,093 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fda96cf025dc,33531,1731522328314' 2024-11-13T18:25:30,093 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T18:25:30,094 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T18:25:30,095 DEBUG [RS:0;fda96cf025dc:33531 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T18:25:30,095 INFO [RS:0;fda96cf025dc:33531 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T18:25:30,095 INFO [RS:0;fda96cf025dc:33531 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T18:25:30,100 WARN [fda96cf025dc:42825 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-11-13T18:25:30,179 INFO [RS:2;fda96cf025dc:41739 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:30,179 INFO [RS:1;fda96cf025dc:44035 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:30,182 INFO [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C44035%2C1731522328435, suffix=, logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435, archiveDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs, maxLogs=32 2024-11-13T18:25:30,182 INFO [RS:2;fda96cf025dc:41739 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C41739%2C1731522328489, suffix=, logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489, archiveDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs, maxLogs=32 2024-11-13T18:25:30,196 INFO [RS:0;fda96cf025dc:33531 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:30,199 INFO [RS:0;fda96cf025dc:33531 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C33531%2C1731522328314, suffix=, logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,33531,1731522328314, archiveDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs, maxLogs=32 2024-11-13T18:25:30,202 DEBUG [RS:1;fda96cf025dc:44035 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.1731522330187, exclude list is [], retry=0 2024-11-13T18:25:30,203 DEBUG [RS:2;fda96cf025dc:41739 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489/fda96cf025dc%2C41739%2C1731522328489.1731522330187, exclude list is [], retry=0 2024-11-13T18:25:30,209 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:30,210 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:30,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:30,211 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:30,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:30,212 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:30,220 DEBUG [RS:0;fda96cf025dc:33531 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,33531,1731522328314/fda96cf025dc%2C33531%2C1731522328314.1731522330201, exclude list is [], retry=0 2024-11-13T18:25:30,222 INFO [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.1731522330187 2024-11-13T18:25:30,223 DEBUG [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:30,223 INFO [RS:2;fda96cf025dc:41739 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489/fda96cf025dc%2C41739%2C1731522328489.1731522330187 2024-11-13T18:25:30,223 DEBUG [RS:2;fda96cf025dc:41739 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:30,224 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:30,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:30,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:30,233 INFO [RS:0;fda96cf025dc:33531 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,33531,1731522328314/fda96cf025dc%2C33531%2C1731522328314.1731522330201 2024-11-13T18:25:30,233 DEBUG [RS:0;fda96cf025dc:33531 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:30,353 DEBUG [fda96cf025dc:42825 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T18:25:30,362 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:25:30,368 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T18:25:30,369 INFO [fda96cf025dc:42825 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:25:30,369 INFO [fda96cf025dc:42825 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:25:30,369 INFO [fda96cf025dc:42825 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T18:25:30,369 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:25:30,376 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:30,383 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,44035,1731522328435, state=OPENING 2024-11-13T18:25:30,388 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T18:25:30,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:30,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:30,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:30,391 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:25:30,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,392 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,393 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T18:25:30,395 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,44035,1731522328435}] 2024-11-13T18:25:30,577 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:25:30,580 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35531, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:25:30,593 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T18:25:30,593 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:30,594 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T18:25:30,597 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fda96cf025dc%2C44035%2C1731522328435.meta, suffix=.meta, logDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435, archiveDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs, maxLogs=32 2024-11-13T18:25:30,613 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.meta.1731522330599.meta, exclude list is [], retry=0 2024-11-13T18:25:30,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:30,617 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:30,618 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:30,621 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.meta.1731522330599.meta 2024-11-13T18:25:30,621 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:30,621 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:30,623 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T18:25:30,626 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T18:25:30,630 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T18:25:30,635 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T18:25:30,635 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:30,635 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T18:25:30,636 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T18:25:30,638 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T18:25:30,640 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T18:25:30,640 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:30,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:30,641 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T18:25:30,642 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T18:25:30,642 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:30,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:30,643 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T18:25:30,644 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T18:25:30,644 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:30,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:30,645 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T18:25:30,646 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T18:25:30,646 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:30,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T18:25:30,647 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T18:25:30,648 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740 2024-11-13T18:25:30,651 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740 2024-11-13T18:25:30,654 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T18:25:30,654 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T18:25:30,655 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T18:25:30,660 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T18:25:30,661 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62612162, jitterRate=-0.06700608134269714}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T18:25:30,662 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T18:25:30,663 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731522330636Writing region info on filesystem at 1731522330636Initializing all the Stores at 1731522330638 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522330638Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522330638Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522330638Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731522330638Cleaning up temporary data from old regions at 1731522330654 (+16 ms)Running coprocessor post-open hooks at 1731522330662 (+8 ms)Region opened successfully at 1731522330663 (+1 ms) 2024-11-13T18:25:30,670 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731522330565 2024-11-13T18:25:30,682 DEBUG [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T18:25:30,682 INFO [RS_OPEN_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T18:25:30,685 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:30,687 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fda96cf025dc,44035,1731522328435, state=OPEN 2024-11-13T18:25:30,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:25:30,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:25:30,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:25:30,690 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T18:25:30,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,690 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T18:25:30,691 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:30,695 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T18:25:30,696 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fda96cf025dc,44035,1731522328435 in 296 msec 2024-11-13T18:25:30,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T18:25:30,702 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 774 msec 2024-11-13T18:25:30,703 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T18:25:30,703 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T18:25:30,748 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:30,749 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44035,1731522328435, seqNum=-1] 2024-11-13T18:25:30,777 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:30,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50993, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:30,800 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2350 sec 2024-11-13T18:25:30,801 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731522330801, completionTime=-1 2024-11-13T18:25:30,803 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T18:25:30,803 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-13T18:25:30,831 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-11-13T18:25:30,831 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731522390831 2024-11-13T18:25:30,831 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731522450831 2024-11-13T18:25:30,831 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 27 msec 2024-11-13T18:25:30,833 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T18:25:30,839 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,839 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,840 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,841 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fda96cf025dc:42825, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,841 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,842 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,848 DEBUG [master/fda96cf025dc:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T18:25:30,870 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.286sec 2024-11-13T18:25:30,872 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T18:25:30,874 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T18:25:30,875 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T18:25:30,876 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T18:25:30,876 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T18:25:30,877 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T18:25:30,877 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T18:25:30,882 DEBUG [master/fda96cf025dc:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T18:25:30,883 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T18:25:30,884 INFO [master/fda96cf025dc:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fda96cf025dc,42825,1731522327393-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:30,949 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a2c045e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:30,950 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fda96cf025dc,42825,-1 for getting cluster id 2024-11-13T18:25:30,953 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T18:25:30,961 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'bd0e7932-69d1-4437-b0c5-be1e6f1fb465' 2024-11-13T18:25:30,963 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T18:25:30,964 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "bd0e7932-69d1-4437-b0c5-be1e6f1fb465" 2024-11-13T18:25:30,966 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a215e37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:30,966 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fda96cf025dc,42825,-1] 2024-11-13T18:25:30,969 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T18:25:30,971 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:30,973 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53378, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T18:25:30,975 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57eb49d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T18:25:30,976 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T18:25:30,982 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fda96cf025dc,44035,1731522328435, seqNum=-1] 2024-11-13T18:25:30,983 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:30,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:59036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:31,005 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fda96cf025dc,42825,1731522327393 2024-11-13T18:25:31,005 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:40367/hbase 2024-11-13T18:25:31,029 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=2534 2024-11-13T18:25:31,058 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:31,061 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:31,063 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:31,073 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-49738988, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-49738988, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:31,094 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-49738988/hregion-49738988.1731522331075, exclude list is [], retry=0 2024-11-13T18:25:31,110 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:31,113 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:31,125 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:31,149 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-49738988/hregion-49738988.1731522331075 2024-11-13T18:25:31,150 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:31,150 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => de93de0511643db93c81c205352fc17a, NAME => 'testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:31,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741838_1014 (size=64) 2024-11-13T18:25:31,167 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741838_1014 (size=64) 2024-11-13T18:25:31,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741838_1014 (size=64) 2024-11-13T18:25:31,169 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:31,171 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,174 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de93de0511643db93c81c205352fc17a columnFamilyName a 2024-11-13T18:25:31,174 DEBUG [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:31,175 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] regionserver.HStore(327): Store=de93de0511643db93c81c205352fc17a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:31,175 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,176 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,177 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,177 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,180 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,183 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:31,185 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de93de0511643db93c81c205352fc17a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60550144, jitterRate=-0.0977325439453125}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:25:31,185 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de93de0511643db93c81c205352fc17a: Writing region info on filesystem at 1731522331169Initializing all the Stores at 1731522331171 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522331171Cleaning up temporary data from old regions at 1731522331177 (+6 ms)Region opened successfully at 1731522331185 (+8 ms) 2024-11-13T18:25:31,186 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing de93de0511643db93c81c205352fc17a, disabling compactions & flushes 2024-11-13T18:25:31,186 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. 2024-11-13T18:25:31,186 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. 2024-11-13T18:25:31,186 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. after waiting 0 ms 2024-11-13T18:25:31,186 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. 2024-11-13T18:25:31,186 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. 2024-11-13T18:25:31,186 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for de93de0511643db93c81c205352fc17a: Waiting for close lock at 1731522331186Disabling compacts and flushes for region at 1731522331186Disabling writes for close at 1731522331186Writing region close event to WAL at 1731522331186Closed at 1731522331186 2024-11-13T18:25:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741837_1013 (size=93) 2024-11-13T18:25:31,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741837_1013 (size=93) 2024-11-13T18:25:31,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741837_1013 (size=93) 2024-11-13T18:25:31,198 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:31,199 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-49738988:(num 1731522331075) 2024-11-13T18:25:31,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T18:25:31,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741839_1015 (size=369) 2024-11-13T18:25:31,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741839_1015 (size=369) 2024-11-13T18:25:31,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741839_1015 (size=369) 2024-11-13T18:25:31,218 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T18:25:31,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741840_1016 (size=231) 2024-11-13T18:25:31,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741840_1016 (size=231) 2024-11-13T18:25:31,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741840_1016 (size=231) 2024-11-13T18:25:31,255 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1, size=369 (369bytes) 2024-11-13T18:25:31,256 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T18:25:31,256 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T18:25:31,256 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1 2024-11-13T18:25:31,262 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1 after 4ms 2024-11-13T18:25:31,268 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1 took 14ms 2024-11-13T18:25:31,272 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1 so closing down 2024-11-13T18:25:31,275 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T18:25:31,278 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:31,278 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T18:25:31,279 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:31,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741841_1017 (size=369) 2024-11-13T18:25:31,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741841_1017 (size=369) 2024-11-13T18:25:31,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741841_1017 (size=369) 2024-11-13T18:25:31,292 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:31,295 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002 2024-11-13T18:25:31,301 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T18:25:31,301 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1, journal: Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1, size=369 (369bytes) at 1731522331255Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1 so closing down at 1731522331272 (+17 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp at 1731522331278 (+6 ms)3 split writer threads finished at 1731522331279 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731522331292 (+13 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002 at 1731522331295 (+3 ms)Processed 2 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731522331301 (+6 ms) 2024-11-13T18:25:31,317 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2, size=231 (231bytes) 2024-11-13T18:25:31,317 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2 2024-11-13T18:25:31,318 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2 after 1ms 2024-11-13T18:25:31,322 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2 took 5ms 2024-11-13T18:25:31,327 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2 so closing down 2024-11-13T18:25:31,327 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:31,330 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T18:25:31,332 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T18:25:31,332 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:31,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741842_1018 (size=231) 2024-11-13T18:25:31,343 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741842_1018 (size=231) 2024-11-13T18:25:31,344 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741842_1018 (size=231) 2024-11-13T18:25:31,345 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:31,351 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-13T18:25:31,353 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T18:25:31,353 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2, journal: Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2, size=231 (231bytes) at 1731522331317Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2 so closing down at 1731522331327 (+10 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002-wal-2.temp at 1731522331332 (+5 ms)3 split writer threads finished at 1731522331332Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731522331345 (+13 ms)Processed 1 edits across 1 Regions in 28 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731522331353 (+8 ms) 2024-11-13T18:25:31,354 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:31,356 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:31,374 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal.1731522331358, exclude list is [], retry=0 2024-11-13T18:25:31,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:31,380 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:31,381 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:31,405 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal.1731522331358 2024-11-13T18:25:31,409 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:31,409 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => de93de0511643db93c81c205352fc17a, NAME => 'testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:31,409 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:31,409 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,409 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,415 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,417 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region de93de0511643db93c81c205352fc17a columnFamilyName a 2024-11-13T18:25:31,417 DEBUG [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:31,418 INFO [StoreOpener-de93de0511643db93c81c205352fc17a-1 {}] regionserver.HStore(327): Store=de93de0511643db93c81c205352fc17a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:31,418 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,420 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,423 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,424 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002 2024-11-13T18:25:31,434 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002 2024-11-13T18:25:31,437 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing de93de0511643db93c81c205352fc17a 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T18:25:31,513 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/.tmp/a/5d659a5311a14289b63ec332c666a317 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731522331199/Put/seqid=0 2024-11-13T18:25:31,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741844_1020 (size=5170) 2024-11-13T18:25:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741844_1020 (size=5170) 2024-11-13T18:25:31,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741844_1020 (size=5170) 2024-11-13T18:25:31,562 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/.tmp/a/5d659a5311a14289b63ec332c666a317 2024-11-13T18:25:31,619 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/.tmp/a/5d659a5311a14289b63ec332c666a317 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/a/5d659a5311a14289b63ec332c666a317 2024-11-13T18:25:31,631 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/a/5d659a5311a14289b63ec332c666a317, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T18:25:31,638 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for de93de0511643db93c81c205352fc17a in 198ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T18:25:31,640 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/0000000000000000002 2024-11-13T18:25:31,641 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,641 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,645 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for de93de0511643db93c81c205352fc17a 2024-11-13T18:25:31,651 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/de93de0511643db93c81c205352fc17a/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T18:25:31,657 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened de93de0511643db93c81c205352fc17a; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67893906, jitterRate=0.011698037385940552}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:25:31,658 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for de93de0511643db93c81c205352fc17a: Writing region info on filesystem at 1731522331410Initializing all the Stores at 1731522331412 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522331412Obtaining lock to block concurrent updates at 1731522331437 (+25 ms)Preparing flush snapshotting stores in de93de0511643db93c81c205352fc17a at 1731522331437Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731522331441 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731522331059.de93de0511643db93c81c205352fc17a. at 1731522331441Flushing de93de0511643db93c81c205352fc17a/a: creating writer at 1731522331443 (+2 ms)Flushing de93de0511643db93c81c205352fc17a/a: appending metadata at 1731522331496 (+53 ms)Flushing de93de0511643db93c81c205352fc17a/a: closing flushed file at 1731522331499 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@568c8d99: reopening flushed file at 1731522331617 (+118 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for de93de0511643db93c81c205352fc17a in 198ms, sequenceid=2, compaction requested=false; wal=null at 1731522331638 (+21 ms)Cleaning up temporary data from old regions at 1731522331641 (+3 ms)Region opened successfully at 1731522331658 (+17 ms) 2024-11-13T18:25:31,697 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 362) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:50096 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:49936 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:39290 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:40367/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:39466 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43486 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43586 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=2515 (was 2534) 2024-11-13T18:25:31,713 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=2514 2024-11-13T18:25:31,729 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:31,731 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:31,732 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:31,735 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45658138, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-45658138, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:31,748 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45658138/hregion-45658138.1731522331735, exclude list is [], retry=0 2024-11-13T18:25:31,751 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:31,751 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:31,751 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:31,754 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45658138/hregion-45658138.1731522331735 2024-11-13T18:25:31,755 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:31,756 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => cfaae1274eea5f5d3d084e9011aff048, NAME => 'testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:31,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741846_1022 (size=64) 2024-11-13T18:25:31,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741846_1022 (size=64) 2024-11-13T18:25:31,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741846_1022 (size=64) 2024-11-13T18:25:31,768 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:31,770 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,773 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfaae1274eea5f5d3d084e9011aff048 columnFamilyName a 2024-11-13T18:25:31,773 DEBUG [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:31,774 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] regionserver.HStore(327): Store=cfaae1274eea5f5d3d084e9011aff048/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:31,774 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,775 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,776 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,777 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,777 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,780 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:31,785 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:31,785 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfaae1274eea5f5d3d084e9011aff048; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61325109, jitterRate=-0.08618466556072235}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:25:31,786 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfaae1274eea5f5d3d084e9011aff048: Writing region info on filesystem at 1731522331768Initializing all the Stores at 1731522331770 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522331770Cleaning up temporary data from old regions at 1731522331777 (+7 ms)Region opened successfully at 1731522331786 (+9 ms) 2024-11-13T18:25:31,786 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing cfaae1274eea5f5d3d084e9011aff048, disabling compactions & flushes 2024-11-13T18:25:31,786 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. 2024-11-13T18:25:31,786 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. 2024-11-13T18:25:31,786 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. after waiting 0 ms 2024-11-13T18:25:31,786 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. 2024-11-13T18:25:31,789 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. 2024-11-13T18:25:31,789 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for cfaae1274eea5f5d3d084e9011aff048: Waiting for close lock at 1731522331786Disabling compacts and flushes for region at 1731522331786Disabling writes for close at 1731522331786Writing region close event to WAL at 1731522331789 (+3 ms)Closed at 1731522331789 2024-11-13T18:25:31,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741845_1021 (size=93) 2024-11-13T18:25:31,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741845_1021 (size=93) 2024-11-13T18:25:31,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741845_1021 (size=93) 2024-11-13T18:25:31,800 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:31,801 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45658138:(num 1731522331735) 2024-11-13T18:25:31,802 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T18:25:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741847_1023 (size=369) 2024-11-13T18:25:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741847_1023 (size=369) 2024-11-13T18:25:31,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741847_1023 (size=369) 2024-11-13T18:25:32,222 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T18:25:32,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741848_1024 (size=231) 2024-11-13T18:25:32,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741848_1024 (size=231) 2024-11-13T18:25:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741848_1024 (size=231) 2024-11-13T18:25:32,258 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2, size=231 (231bytes) 2024-11-13T18:25:32,258 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2 2024-11-13T18:25:32,259 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2 after 1ms 2024-11-13T18:25:32,264 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2 took 7ms 2024-11-13T18:25:32,266 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2 so closing down 2024-11-13T18:25:32,266 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:32,268 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T18:25:32,270 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T18:25:32,270 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:32,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741849_1025 (size=231) 2024-11-13T18:25:32,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741849_1025 (size=231) 2024-11-13T18:25:32,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741849_1025 (size=231) 2024-11-13T18:25:32,281 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:32,283 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 2024-11-13T18:25:32,283 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T18:25:32,283 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2, journal: Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2, size=231 (231bytes) at 1731522332258Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2 so closing down at 1731522332266 (+8 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp at 1731522332270 (+4 ms)3 split writer threads finished at 1731522332270Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731522332281 (+11 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 at 1731522332283 (+2 ms)Processed 1 edits across 1 Regions in 19 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731522332283 2024-11-13T18:25:32,307 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1, size=369 (369bytes) 2024-11-13T18:25:32,307 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1 2024-11-13T18:25:32,308 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1 after 1ms 2024-11-13T18:25:32,312 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1 took 6ms 2024-11-13T18:25:32,314 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1 so closing down 2024-11-13T18:25:32,314 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:32,317 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T18:25:32,319 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T18:25:32,319 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:32,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741850_1026 (size=369) 2024-11-13T18:25:32,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741850_1026 (size=369) 2024-11-13T18:25:32,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741850_1026 (size=369) 2024-11-13T18:25:32,342 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:32,349 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002, length=231 2024-11-13T18:25:32,352 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 2024-11-13T18:25:32,353 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T18:25:32,353 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1, journal: Splitting hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1, size=369 (369bytes) at 1731522332307Finishing writing output for hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1 so closing down at 1731522332314 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp at 1731522332319 (+5 ms)3 split writer threads finished at 1731522332319Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731522332342 (+23 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 at 1731522332352 (+10 ms)Processed 2 edits across 1 Regions in 40 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731522332353 (+1 ms) 2024-11-13T18:25:32,353 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:32,356 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:32,374 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal.1731522332356, exclude list is [], retry=0 2024-11-13T18:25:32,378 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:32,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:32,379 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:32,383 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal.1731522332356 2024-11-13T18:25:32,384 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:32,384 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => cfaae1274eea5f5d3d084e9011aff048, NAME => 'testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:32,384 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:32,384 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,384 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,388 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,389 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cfaae1274eea5f5d3d084e9011aff048 columnFamilyName a 2024-11-13T18:25:32,390 DEBUG [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:32,390 INFO [StoreOpener-cfaae1274eea5f5d3d084e9011aff048-1 {}] regionserver.HStore(327): Store=cfaae1274eea5f5d3d084e9011aff048/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:32,391 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,391 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,394 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,395 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 2024-11-13T18:25:32,401 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 2024-11-13T18:25:32,401 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing cfaae1274eea5f5d3d084e9011aff048 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T18:25:32,437 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/.tmp/a/401b6af374344244876a36339a2893b1 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731522331801/Put/seqid=0 2024-11-13T18:25:32,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741852_1028 (size=5170) 2024-11-13T18:25:32,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741852_1028 (size=5170) 2024-11-13T18:25:32,459 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741852_1028 (size=5170) 2024-11-13T18:25:32,459 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/.tmp/a/401b6af374344244876a36339a2893b1 2024-11-13T18:25:32,471 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/.tmp/a/401b6af374344244876a36339a2893b1 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/a/401b6af374344244876a36339a2893b1 2024-11-13T18:25:32,479 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/a/401b6af374344244876a36339a2893b1, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T18:25:32,479 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cfaae1274eea5f5d3d084e9011aff048 in 78ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T18:25:32,480 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/0000000000000000002 2024-11-13T18:25:32,480 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,480 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,483 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for cfaae1274eea5f5d3d084e9011aff048 2024-11-13T18:25:32,485 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/cfaae1274eea5f5d3d084e9011aff048/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T18:25:32,486 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened cfaae1274eea5f5d3d084e9011aff048; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72798735, jitterRate=0.08478568494319916}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:25:32,487 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for cfaae1274eea5f5d3d084e9011aff048: Writing region info on filesystem at 1731522332385Initializing all the Stores at 1731522332386 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522332386Obtaining lock to block concurrent updates at 1731522332401 (+15 ms)Preparing flush snapshotting stores in cfaae1274eea5f5d3d084e9011aff048 at 1731522332401Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731522332401Flushing stores of testReplayEditsWrittenIntoWAL,,1731522331729.cfaae1274eea5f5d3d084e9011aff048. at 1731522332401Flushing cfaae1274eea5f5d3d084e9011aff048/a: creating writer at 1731522332401Flushing cfaae1274eea5f5d3d084e9011aff048/a: appending metadata at 1731522332435 (+34 ms)Flushing cfaae1274eea5f5d3d084e9011aff048/a: closing flushed file at 1731522332435Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a302229: reopening flushed file at 1731522332469 (+34 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for cfaae1274eea5f5d3d084e9011aff048 in 78ms, sequenceid=2, compaction requested=false; wal=null at 1731522332479 (+10 ms)Cleaning up temporary data from old regions at 1731522332480 (+1 ms)Region opened successfully at 1731522332487 (+7 ms) 2024-11-13T18:25:32,503 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:50178 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:49936 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:39290 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43486 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:39538 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43648 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=490 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=2529 (was 2514) - AvailableMemoryMB LEAK? - 2024-11-13T18:25:32,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=490, ProcessCount=11, AvailableMemoryMB=2528 2024-11-13T18:25:32,533 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:32,535 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:32,536 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:32,540 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-57027590, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-57027590, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:32,554 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-57027590/hregion-57027590.1731522332540, exclude list is [], retry=0 2024-11-13T18:25:32,557 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:32,557 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:32,558 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:32,560 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-57027590/hregion-57027590.1731522332540 2024-11-13T18:25:32,562 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:32,562 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1e4f51a939ebe8a5287bf7a4e5a20272, NAME => 'testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:32,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741854_1030 (size=64) 2024-11-13T18:25:32,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741854_1030 (size=64) 2024-11-13T18:25:32,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741854_1030 (size=64) 2024-11-13T18:25:32,577 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:32,579 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,580 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName a 2024-11-13T18:25:32,581 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:32,581 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:32,581 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,583 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName b 2024-11-13T18:25:32,583 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:32,584 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:32,584 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,585 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName c 2024-11-13T18:25:32,585 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:32,586 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:32,586 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,587 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,588 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,589 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,589 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,590 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:32,591 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:32,593 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:32,594 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e4f51a939ebe8a5287bf7a4e5a20272; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69076503, jitterRate=0.02932010591030121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e4f51a939ebe8a5287bf7a4e5a20272: Writing region info on filesystem at 1731522332577Initializing all the Stores at 1731522332578 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522332578Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522332579 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522332579Cleaning up temporary data from old regions at 1731522332589 (+10 ms)Region opened successfully at 1731522332594 (+5 ms) 2024-11-13T18:25:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e4f51a939ebe8a5287bf7a4e5a20272, disabling compactions & flushes 2024-11-13T18:25:32,595 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. after waiting 0 ms 2024-11-13T18:25:32,595 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:32,595 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:32,596 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e4f51a939ebe8a5287bf7a4e5a20272: Waiting for close lock at 1731522332595Disabling compacts and flushes for region at 1731522332595Disabling writes for close at 1731522332595Writing region close event to WAL at 1731522332595Closed at 1731522332595 2024-11-13T18:25:32,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741853_1029 (size=93) 2024-11-13T18:25:32,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741853_1029 (size=93) 2024-11-13T18:25:32,599 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741853_1029 (size=93) 2024-11-13T18:25:32,605 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:32,605 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-57027590:(num 1731522332540) 2024-11-13T18:25:32,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:32,608 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:32,622 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, exclude list is [], retry=0 2024-11-13T18:25:32,625 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:32,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:32,626 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:32,630 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 2024-11-13T18:25:32,630 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:32,833 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, size=0 (0bytes) 2024-11-13T18:25:32,833 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 might be still open, length is 0 2024-11-13T18:25:32,833 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 2024-11-13T18:25:32,834 WARN [IPC Server handler 1 on default port 40367 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T18:25:32,835 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 after 2ms 2024-11-13T18:25:35,234 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43686 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43686 dst: /127.0.0.1:41041 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41041 remote=/127.0.0.1:43686]. Total timeout mills is 60000, 57555 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:35,235 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:50206 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:33523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50206 dst: /127.0.0.1:33523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:35,236 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:39580 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:46817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39580 dst: /127.0.0.1:46817 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:35,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741855_1032 (size=470675) 2024-11-13T18:25:35,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741855_1032 (size=470675) 2024-11-13T18:25:35,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741855_1032 (size=470675) 2024-11-13T18:25:36,144 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:25:36,188 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T18:25:36,836 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 after 4003ms 2024-11-13T18:25:36,842 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 took 4010ms 2024-11-13T18:25:36,847 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731522332608.temp 2024-11-13T18:25:36,852 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp 2024-11-13T18:25:36,940 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608; continuing. 2024-11-13T18:25:36,940 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 so closing down 2024-11-13T18:25:36,941 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:36,944 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:36,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741856_1033 (size=470683) 2024-11-13T18:25:36,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741856_1033 (size=470683) 2024-11-13T18:25:36,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741856_1033 (size=470683) 2024-11-13T18:25:36,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp (wrote 3002 edits, skipped 0 edits in 29 ms) 2024-11-13T18:25:36,957 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 2024-11-13T18:25:36,957 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 115 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T18:25:36,957 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, journal: Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, size=0 (0bytes) at 1731522332833Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp at 1731522336853 (+4020 ms)Split 1024 edits, skipped 0 edits. at 1731522336900 (+47 ms)Split 2048 edits, skipped 0 edits. at 1731522336922 (+22 ms)Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 so closing down at 1731522336940 (+18 ms)3 split writer threads finished at 1731522336944 (+4 ms)Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp (wrote 3002 edits, skipped 0 edits in 29 ms) at 1731522336954 (+10 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000000001-wal.1731522332608.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 at 1731522336957 (+3 ms)Processed 3002 edits across 1 Regions in 115 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608, size=0, length=0, corrupted=false, cancelled=false at 1731522336957 2024-11-13T18:25:36,960 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522332608 2024-11-13T18:25:36,961 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 2024-11-13T18:25:36,962 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:36,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:36,984 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522336965, exclude list is [], retry=0 2024-11-13T18:25:36,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:36,988 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:36,989 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:37,001 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522336965 2024-11-13T18:25:37,002 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:37,002 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:37,011 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,013 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName a 2024-11-13T18:25:37,013 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,014 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:37,014 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,015 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName b 2024-11-13T18:25:37,016 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,016 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:37,016 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,018 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1e4f51a939ebe8a5287bf7a4e5a20272 columnFamilyName c 2024-11-13T18:25:37,018 DEBUG [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:37,018 INFO [StoreOpener-1e4f51a939ebe8a5287bf7a4e5a20272-1 {}] regionserver.HStore(327): Store=1e4f51a939ebe8a5287bf7a4e5a20272/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:37,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,024 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:37,027 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 2024-11-13T18:25:37,095 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T18:25:37,522 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e4f51a939ebe8a5287bf7a4e5a20272 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T18:25:37,588 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/c6e5c66812b440b984a97f545b5a0ec1 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731522332637/Put/seqid=0 2024-11-13T18:25:37,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741858_1035 (size=50463) 2024-11-13T18:25:37,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741858_1035 (size=50463) 2024-11-13T18:25:37,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741858_1035 (size=50463) 2024-11-13T18:25:37,606 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/c6e5c66812b440b984a97f545b5a0ec1 2024-11-13T18:25:37,614 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/c6e5c66812b440b984a97f545b5a0ec1 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/a/c6e5c66812b440b984a97f545b5a0ec1 2024-11-13T18:25:37,622 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/a/c6e5c66812b440b984a97f545b5a0ec1, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T18:25:37,622 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 1e4f51a939ebe8a5287bf7a4e5a20272 in 100ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T18:25:37,638 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T18:25:37,638 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e4f51a939ebe8a5287bf7a4e5a20272 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T18:25:37,652 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/4e8a13aae68040c2b6afc9cfe4b83395 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731522332678/Put/seqid=0 2024-11-13T18:25:37,668 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741859_1036 (size=20072) 2024-11-13T18:25:37,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741859_1036 (size=20072) 2024-11-13T18:25:37,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741859_1036 (size=20072) 2024-11-13T18:25:37,671 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/4e8a13aae68040c2b6afc9cfe4b83395 2024-11-13T18:25:37,703 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/a6cd29385f034be9b7a3f8d59601aeaf is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731522332699/Put/seqid=0 2024-11-13T18:25:37,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741860_1037 (size=35835) 2024-11-13T18:25:37,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741860_1037 (size=35835) 2024-11-13T18:25:37,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741860_1037 (size=35835) 2024-11-13T18:25:37,715 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/a6cd29385f034be9b7a3f8d59601aeaf 2024-11-13T18:25:37,723 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/a/4e8a13aae68040c2b6afc9cfe4b83395 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/a/4e8a13aae68040c2b6afc9cfe4b83395 2024-11-13T18:25:37,731 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/a/4e8a13aae68040c2b6afc9cfe4b83395, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T18:25:37,733 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/a6cd29385f034be9b7a3f8d59601aeaf as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/b/a6cd29385f034be9b7a3f8d59601aeaf 2024-11-13T18:25:37,741 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/b/a6cd29385f034be9b7a3f8d59601aeaf, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T18:25:37,741 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1e4f51a939ebe8a5287bf7a4e5a20272 in 103ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T18:25:37,752 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T18:25:37,752 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e4f51a939ebe8a5287bf7a4e5a20272 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T18:25:37,764 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/3f75d57aa72b42f6834296a98552c738 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731522332718/Put/seqid=0 2024-11-13T18:25:37,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741861_1038 (size=35082) 2024-11-13T18:25:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741861_1038 (size=35082) 2024-11-13T18:25:37,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741861_1038 (size=35082) 2024-11-13T18:25:37,788 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/3f75d57aa72b42f6834296a98552c738 2024-11-13T18:25:37,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/6873a0187e4542fc817c672a5a4a2c05 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731522332749/Put/seqid=0 2024-11-13T18:25:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741862_1039 (size=20825) 2024-11-13T18:25:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741862_1039 (size=20825) 2024-11-13T18:25:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741862_1039 (size=20825) 2024-11-13T18:25:37,849 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/6873a0187e4542fc817c672a5a4a2c05 2024-11-13T18:25:37,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/b/3f75d57aa72b42f6834296a98552c738 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/b/3f75d57aa72b42f6834296a98552c738 2024-11-13T18:25:37,879 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T18:25:37,879 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:37,882 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:25:37,882 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T18:25:37,883 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T18:25:37,883 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:37,883 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/b/3f75d57aa72b42f6834296a98552c738, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T18:25:37,884 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T18:25:37,884 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:37,885 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/6873a0187e4542fc817c672a5a4a2c05 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/c/6873a0187e4542fc817c672a5a4a2c05 2024-11-13T18:25:37,893 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/c/6873a0187e4542fc817c672a5a4a2c05, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T18:25:37,894 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 1e4f51a939ebe8a5287bf7a4e5a20272 in 142ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T18:25:37,902 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731522332788/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:37,907 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 2024-11-13T18:25:37,907 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T18:25:37,908 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1e4f51a939ebe8a5287bf7a4e5a20272 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T18:25:37,926 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/fb6318f5512e46868225edb827f5f53b is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731522332755/Put/seqid=0 2024-11-13T18:25:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741863_1040 (size=50301) 2024-11-13T18:25:37,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741863_1040 (size=50301) 2024-11-13T18:25:37,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741863_1040 (size=50301) 2024-11-13T18:25:38,362 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/fb6318f5512e46868225edb827f5f53b 2024-11-13T18:25:38,378 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb6318f5512e46868225edb827f5f53b 2024-11-13T18:25:38,382 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/.tmp/c/fb6318f5512e46868225edb827f5f53b as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/c/fb6318f5512e46868225edb827f5f53b 2024-11-13T18:25:38,391 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb6318f5512e46868225edb827f5f53b 2024-11-13T18:25:38,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/c/fb6318f5512e46868225edb827f5f53b, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T18:25:38,392 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 1e4f51a939ebe8a5287bf7a4e5a20272 in 485ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T18:25:38,393 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/0000000000000003002 2024-11-13T18:25:38,395 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:38,395 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:38,396 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T18:25:38,398 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1e4f51a939ebe8a5287bf7a4e5a20272 2024-11-13T18:25:38,403 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenIntoWAL/1e4f51a939ebe8a5287bf7a4e5a20272/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T18:25:38,405 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1e4f51a939ebe8a5287bf7a4e5a20272; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61333071, jitterRate=-0.08606602251529694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T18:25:38,405 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1e4f51a939ebe8a5287bf7a4e5a20272: Writing region info on filesystem at 1731522337002Initializing all the Stores at 1731522337004 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522337004Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522337011 (+7 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522337011Cleaning up temporary data from old regions at 1731522338395 (+1384 ms)Region opened successfully at 1731522338405 (+10 ms) 2024-11-13T18:25:38,558 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1e4f51a939ebe8a5287bf7a4e5a20272, disabling compactions & flushes 2024-11-13T18:25:38,558 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:38,558 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:38,558 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. after waiting 0 ms 2024-11-13T18:25:38,558 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:38,573 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731522332534.1e4f51a939ebe8a5287bf7a4e5a20272. 2024-11-13T18:25:38,573 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1e4f51a939ebe8a5287bf7a4e5a20272: Waiting for close lock at 1731522338558Disabling compacts and flushes for region at 1731522338558Disabling writes for close at 1731522338558Writing region close event to WAL at 1731522338573 (+15 ms)Closed at 1731522338573 2024-11-13T18:25:38,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741857_1034 (size=93) 2024-11-13T18:25:38,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741857_1034 (size=93) 2024-11-13T18:25:38,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741857_1034 (size=93) 2024-11-13T18:25:38,585 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:38,585 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731522336965) 2024-11-13T18:25:38,611 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 383) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:36759 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:40367 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:36759 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-154851637_22 at /127.0.0.1:56018 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:41289 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40367 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-154851637_22 at /127.0.0.1:35178 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41289 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-154851637_22 at /127.0.0.1:55996 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-154851637_22 at /127.0.0.1:43616 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=514 (was 490) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3139 (was 2528) - AvailableMemoryMB LEAK? - 2024-11-13T18:25:38,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=514, ProcessCount=11, AvailableMemoryMB=3139 2024-11-13T18:25:38,647 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:38,649 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:38,650 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:38,653 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-69565713, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-69565713, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:38,666 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-69565713/hregion-69565713.1731522338653, exclude list is [], retry=0 2024-11-13T18:25:38,669 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:38,673 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:38,681 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:38,683 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-69565713/hregion-69565713.1731522338653 2024-11-13T18:25:38,684 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:38,684 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e1130e6d07a7307d7e5c9cd481ecfb8a, NAME => 'test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741865_1042 (size=43) 2024-11-13T18:25:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741865_1042 (size=43) 2024-11-13T18:25:38,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741865_1042 (size=43) 2024-11-13T18:25:38,697 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:38,699 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,700 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName a 2024-11-13T18:25:38,701 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,701 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:38,701 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,703 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName b 2024-11-13T18:25:38,703 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,703 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:38,704 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,705 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName c 2024-11-13T18:25:38,705 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:38,706 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:38,706 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,706 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,707 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,708 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,708 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,709 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:38,710 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:38,712 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:38,713 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e1130e6d07a7307d7e5c9cd481ecfb8a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60653397, jitterRate=-0.09619395434856415}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e1130e6d07a7307d7e5c9cd481ecfb8a: Writing region info on filesystem at 1731522338697Initializing all the Stores at 1731522338698 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522338698Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522338699 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522338699Cleaning up temporary data from old regions at 1731522338708 (+9 ms)Region opened successfully at 1731522338714 (+6 ms) 2024-11-13T18:25:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e1130e6d07a7307d7e5c9cd481ecfb8a, disabling compactions & flushes 2024-11-13T18:25:38,714 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. after waiting 0 ms 2024-11-13T18:25:38,714 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:38,715 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:38,715 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e1130e6d07a7307d7e5c9cd481ecfb8a: Waiting for close lock at 1731522338714Disabling compacts and flushes for region at 1731522338714Disabling writes for close at 1731522338714Writing region close event to WAL at 1731522338715 (+1 ms)Closed at 1731522338715 2024-11-13T18:25:38,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741864_1041 (size=93) 2024-11-13T18:25:38,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741864_1041 (size=93) 2024-11-13T18:25:38,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741864_1041 (size=93) 2024-11-13T18:25:38,722 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:38,722 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-69565713:(num 1731522338653) 2024-11-13T18:25:38,722 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:38,724 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:38,738 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, exclude list is [], retry=0 2024-11-13T18:25:38,741 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:38,742 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:38,742 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:38,744 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 2024-11-13T18:25:38,747 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:38,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741866_1043 (size=344306) 2024-11-13T18:25:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741866_1043 (size=344306) 2024-11-13T18:25:38,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741866_1043 (size=344306) 2024-11-13T18:25:38,932 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 not finished, retry = 0 2024-11-13T18:25:39,071 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, size=336.2 K (344306bytes) 2024-11-13T18:25:39,071 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 2024-11-13T18:25:39,072 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 after 0ms 2024-11-13T18:25:39,076 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 took 5ms 2024-11-13T18:25:39,089 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731522338724.temp 2024-11-13T18:25:39,092 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp 2024-11-13T18:25:39,132 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 so closing down 2024-11-13T18:25:39,132 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:39,133 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:39,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741867_1044 (size=344306) 2024-11-13T18:25:39,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741867_1044 (size=344306) 2024-11-13T18:25:39,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741867_1044 (size=344306) 2024-11-13T18:25:39,159 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp (wrote 3000 edits, skipped 0 edits in 12 ms) 2024-11-13T18:25:39,162 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp to hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 2024-11-13T18:25:39,162 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-13T18:25:39,162 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, journal: Splitting hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, size=336.2 K (344306bytes) at 1731522339071Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp at 1731522339093 (+22 ms)Split 1024 edits, skipped 0 edits. at 1731522339099 (+6 ms)Split 2048 edits, skipped 0 edits. at 1731522339120 (+21 ms)Finishing writing output for hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 so closing down at 1731522339132 (+12 ms)3 split writer threads finished at 1731522339133 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp (wrote 3000 edits, skipped 0 edits in 12 ms) at 1731522339159 (+26 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000000001-wal.1731522338724.temp to hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 at 1731522339162 (+3 ms)Processed 3000 edits across 1 Regions in 85 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731522339162 2024-11-13T18:25:39,174 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522338724 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522338724 2024-11-13T18:25:39,184 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 2024-11-13T18:25:39,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:39,191 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:39,214 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, exclude list is [], retry=0 2024-11-13T18:25:39,223 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:39,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:39,226 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:39,241 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 2024-11-13T18:25:39,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741868_1045 (size=344433) 2024-11-13T18:25:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741868_1045 (size=344433) 2024-11-13T18:25:39,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741868_1045 (size=344433) 2024-11-13T18:25:39,468 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 not finished, retry = 0 2024-11-13T18:25:39,592 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, size=336.4 K (344433bytes) 2024-11-13T18:25:39,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 2024-11-13T18:25:39,592 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 after 0ms 2024-11-13T18:25:39,596 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 took 5ms 2024-11-13T18:25:39,607 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731522339192.temp 2024-11-13T18:25:39,609 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp 2024-11-13T18:25:39,633 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 so closing down 2024-11-13T18:25:39,633 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:39,636 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:39,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741869_1046 (size=344433) 2024-11-13T18:25:39,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741869_1046 (size=344433) 2024-11-13T18:25:39,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741869_1046 (size=344433) 2024-11-13T18:25:40,044 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-13T18:25:40,046 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp to hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 2024-11-13T18:25:40,046 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 448 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-13T18:25:40,046 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, journal: Splitting hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, size=336.4 K (344433bytes) at 1731522339592Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp at 1731522339609 (+17 ms)Split 1024 edits, skipped 0 edits. at 1731522339611 (+2 ms)Split 2048 edits, skipped 0 edits. at 1731522339625 (+14 ms)Finishing writing output for hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 so closing down at 1731522339633 (+8 ms)3 split writer threads finished at 1731522339636 (+3 ms)Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731522340044 (+408 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003001-wal.1731522339192.temp to hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 at 1731522340046 (+2 ms)Processed 3000 edits across 1 Regions in 448 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731522340046 2024-11-13T18:25:40,048 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522339192 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522339192 2024-11-13T18:25:40,049 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 2024-11-13T18:25:40,050 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:40,052 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/test2727-manual,16010,1731522338646, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:40,067 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522340052, exclude list is [], retry=0 2024-11-13T18:25:40,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:40,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:40,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:40,073 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731522338646/wal.1731522340052 2024-11-13T18:25:40,073 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:40,073 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e1130e6d07a7307d7e5c9cd481ecfb8a, NAME => 'test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:40,073 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:40,073 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,074 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,075 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,076 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName a 2024-11-13T18:25:40,076 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:40,076 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:40,077 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,078 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName b 2024-11-13T18:25:40,078 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:40,078 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:40,078 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,079 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1130e6d07a7307d7e5c9cd481ecfb8a columnFamilyName c 2024-11-13T18:25:40,079 DEBUG [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:40,080 INFO [StoreOpener-e1130e6d07a7307d7e5c9cd481ecfb8a-1 {}] regionserver.HStore(327): Store=e1130e6d07a7307d7e5c9cd481ecfb8a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:40,080 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,080 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,083 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 2024-11-13T18:25:40,116 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 2024-11-13T18:25:40,117 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 2024-11-13T18:25:40,161 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 2024-11-13T18:25:40,161 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e1130e6d07a7307d7e5c9cd481ecfb8a 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T18:25:40,194 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/a/4c973127b10a4d298a2e190156d2fd33 is 41, key is test2727/a:100/1731522339249/Put/seqid=0 2024-11-13T18:25:40,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741871_1048 (size=84227) 2024-11-13T18:25:40,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741871_1048 (size=84227) 2024-11-13T18:25:40,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741871_1048 (size=84227) 2024-11-13T18:25:40,210 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/a/4c973127b10a4d298a2e190156d2fd33 2024-11-13T18:25:40,255 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/b/0e276e2929dc40f2815f36f7ff7efd35 is 41, key is test2727/b:100/1731522339311/Put/seqid=0 2024-11-13T18:25:40,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741872_1049 (size=84609) 2024-11-13T18:25:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741872_1049 (size=84609) 2024-11-13T18:25:40,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741872_1049 (size=84609) 2024-11-13T18:25:40,670 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/b/0e276e2929dc40f2815f36f7ff7efd35 2024-11-13T18:25:40,708 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/c/e233c07a6779498eb2ca1322aa2f63c3 is 41, key is test2727/c:100/1731522339371/Put/seqid=0 2024-11-13T18:25:40,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741873_1050 (size=84609) 2024-11-13T18:25:40,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741873_1050 (size=84609) 2024-11-13T18:25:40,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741873_1050 (size=84609) 2024-11-13T18:25:40,719 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/c/e233c07a6779498eb2ca1322aa2f63c3 2024-11-13T18:25:40,728 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/a/4c973127b10a4d298a2e190156d2fd33 as hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/a/4c973127b10a4d298a2e190156d2fd33 2024-11-13T18:25:40,735 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/a/4c973127b10a4d298a2e190156d2fd33, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T18:25:40,736 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/b/0e276e2929dc40f2815f36f7ff7efd35 as hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/b/0e276e2929dc40f2815f36f7ff7efd35 2024-11-13T18:25:40,744 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/b/0e276e2929dc40f2815f36f7ff7efd35, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T18:25:40,745 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/.tmp/c/e233c07a6779498eb2ca1322aa2f63c3 as hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/c/e233c07a6779498eb2ca1322aa2f63c3 2024-11-13T18:25:40,753 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/c/e233c07a6779498eb2ca1322aa2f63c3, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T18:25:40,753 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e1130e6d07a7307d7e5c9cd481ecfb8a in 592ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T18:25:40,754 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000003000 2024-11-13T18:25:40,755 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/0000000000000006000 2024-11-13T18:25:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,756 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,757 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:40,759 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e1130e6d07a7307d7e5c9cd481ecfb8a 2024-11-13T18:25:40,765 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/test2727/e1130e6d07a7307d7e5c9cd481ecfb8a/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T18:25:40,766 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e1130e6d07a7307d7e5c9cd481ecfb8a; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64175057, jitterRate=-0.04371713101863861}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:40,768 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e1130e6d07a7307d7e5c9cd481ecfb8a: Writing region info on filesystem at 1731522340074Initializing all the Stores at 1731522340074Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522340075 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522340075Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522340075Obtaining lock to block concurrent updates at 1731522340161 (+86 ms)Preparing flush snapshotting stores in e1130e6d07a7307d7e5c9cd481ecfb8a at 1731522340161Finished memstore snapshotting test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731522340162 (+1 ms)Flushing stores of test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. at 1731522340162Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/a: creating writer at 1731522340162Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/a: appending metadata at 1731522340193 (+31 ms)Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/a: closing flushed file at 1731522340193Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/b: creating writer at 1731522340223 (+30 ms)Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/b: appending metadata at 1731522340253 (+30 ms)Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/b: closing flushed file at 1731522340253Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/c: creating writer at 1731522340683 (+430 ms)Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/c: appending metadata at 1731522340707 (+24 ms)Flushing e1130e6d07a7307d7e5c9cd481ecfb8a/c: closing flushed file at 1731522340707Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@54e76f2: reopening flushed file at 1731522340726 (+19 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d4bc453: reopening flushed file at 1731522340735 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4858bbc4: reopening flushed file at 1731522340744 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e1130e6d07a7307d7e5c9cd481ecfb8a in 592ms, sequenceid=6000, compaction requested=false; wal=null at 1731522340753 (+9 ms)Cleaning up temporary data from old regions at 1731522340756 (+3 ms)Region opened successfully at 1731522340768 (+12 ms) 2024-11-13T18:25:40,770 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T18:25:40,770 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e1130e6d07a7307d7e5c9cd481ecfb8a, disabling compactions & flushes 2024-11-13T18:25:40,770 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:40,770 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:40,770 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. after waiting 0 ms 2024-11-13T18:25:40,770 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:40,773 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731522338648.e1130e6d07a7307d7e5c9cd481ecfb8a. 2024-11-13T18:25:40,773 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e1130e6d07a7307d7e5c9cd481ecfb8a: Waiting for close lock at 1731522340770Disabling compacts and flushes for region at 1731522340770Disabling writes for close at 1731522340770Writing region close event to WAL at 1731522340773 (+3 ms)Closed at 1731522340773 2024-11-13T18:25:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741870_1047 (size=93) 2024-11-13T18:25:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741870_1047 (size=93) 2024-11-13T18:25:40,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741870_1047 (size=93) 2024-11-13T18:25:40,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:40,781 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731522340052) 2024-11-13T18:25:40,798 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=398 (was 399), OpenFileDescriptor=923 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=514 (was 514), ProcessCount=11 (was 11), AvailableMemoryMB=2904 (was 3139) 2024-11-13T18:25:40,809 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=398, OpenFileDescriptor=923, MaxFileDescriptor=1048576, SystemLoadAverage=514, ProcessCount=11, AvailableMemoryMB=2902 2024-11-13T18:25:40,826 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:40,833 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:40,834 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731522340833 2024-11-13T18:25:40,843 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 2024-11-13T18:25:40,849 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:40,851 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ec22669737be3720bfdde009ed7a1ff9, NAME => 'testSequentialEditLogSeqNum,,1731522340827.ec22669737be3720bfdde009ed7a1ff9.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:40,851 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731522340827.ec22669737be3720bfdde009ed7a1ff9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:40,851 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:40,851 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:40,853 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9 doesn't exist for region: ec22669737be3720bfdde009ed7a1ff9 on table testSequentialEditLogSeqNum 2024-11-13T18:25:40,853 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: ec22669737be3720bfdde009ed7a1ff9 on table testSequentialEditLogSeqNum 2024-11-13T18:25:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741875_1052 (size=62) 2024-11-13T18:25:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741875_1052 (size=62) 2024-11-13T18:25:40,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741875_1052 (size=62) 2024-11-13T18:25:41,265 INFO [StoreOpener-ec22669737be3720bfdde009ed7a1ff9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,267 INFO [StoreOpener-ec22669737be3720bfdde009ed7a1ff9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ec22669737be3720bfdde009ed7a1ff9 columnFamilyName a 2024-11-13T18:25:41,267 DEBUG [StoreOpener-ec22669737be3720bfdde009ed7a1ff9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:41,268 INFO [StoreOpener-ec22669737be3720bfdde009ed7a1ff9-1 {}] regionserver.HStore(327): Store=ec22669737be3720bfdde009ed7a1ff9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:41,268 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,269 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,270 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,270 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,270 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,272 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ec22669737be3720bfdde009ed7a1ff9 2024-11-13T18:25:41,275 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:41,275 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ec22669737be3720bfdde009ed7a1ff9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61978012, jitterRate=-0.07645565271377563}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:25:41,277 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ec22669737be3720bfdde009ed7a1ff9: Writing region info on filesystem at 1731522340852Initializing all the Stores at 1731522341264 (+412 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522341264Cleaning up temporary data from old regions at 1731522341270 (+6 ms)Region opened successfully at 1731522341277 (+7 ms) 2024-11-13T18:25:41,290 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ec22669737be3720bfdde009ed7a1ff9 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T18:25:41,319 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/.tmp/a/e0ec2375e4374722aab2c80de989ff7e is 81, key is testSequentialEditLogSeqNum/a:x0/1731522341277/Put/seqid=0 2024-11-13T18:25:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741876_1053 (size=5833) 2024-11-13T18:25:41,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741876_1053 (size=5833) 2024-11-13T18:25:41,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741876_1053 (size=5833) 2024-11-13T18:25:41,328 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/.tmp/a/e0ec2375e4374722aab2c80de989ff7e 2024-11-13T18:25:41,335 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/.tmp/a/e0ec2375e4374722aab2c80de989ff7e as hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/a/e0ec2375e4374722aab2c80de989ff7e 2024-11-13T18:25:41,341 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/a/e0ec2375e4374722aab2c80de989ff7e, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T18:25:41,343 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ec22669737be3720bfdde009ed7a1ff9 in 54ms, sequenceid=13, compaction requested=false 2024-11-13T18:25:41,343 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ec22669737be3720bfdde009ed7a1ff9: 2024-11-13T18:25:41,349 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:25:41,349 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:25:41,350 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:25:41,350 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:25:41,350 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T18:25:41,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741874_1051 (size=3017) 2024-11-13T18:25:41,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741874_1051 (size=3017) 2024-11-13T18:25:41,354 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741874_1051 (size=3017) 2024-11-13T18:25:41,379 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833, size=2.9 K (3017bytes) 2024-11-13T18:25:41,379 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 2024-11-13T18:25:41,380 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 after 1ms 2024-11-13T18:25:41,383 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 took 4ms 2024-11-13T18:25:41,386 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 so closing down 2024-11-13T18:25:41,386 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:41,388 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731522340833.temp 2024-11-13T18:25:41,389 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp 2024-11-13T18:25:41,389 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741877_1054 (size=2433) 2024-11-13T18:25:41,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741877_1054 (size=2433) 2024-11-13T18:25:41,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741877_1054 (size=2433) 2024-11-13T18:25:41,402 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:41,404 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp to hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000020 2024-11-13T18:25:41,404 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-13T18:25:41,404 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833, journal: Splitting hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833, size=2.9 K (3017bytes) at 1731522341379Finishing writing output for hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833 so closing down at 1731522341386 (+7 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp at 1731522341389 (+3 ms)3 split writer threads finished at 1731522341390 (+1 ms)Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731522341402 (+12 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000003-wal.1731522340833.temp to hdfs://localhost:40367/hbase/data/default/testSequentialEditLogSeqNum/ec22669737be3720bfdde009ed7a1ff9/recovered.edits/0000000000000000020 at 1731522341404 (+2 ms)Processed 17 edits across 1 Regions in 20 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731522340825/wal.1731522340833, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731522341404 2024-11-13T18:25:41,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=405 (was 398) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:56018 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43692 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:55996 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=961 (was 923) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=514 (was 514), ProcessCount=11 (was 11), AvailableMemoryMB=2883 (was 2902) 2024-11-13T18:25:41,441 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=405, OpenFileDescriptor=961, MaxFileDescriptor=1048576, SystemLoadAverage=514, ProcessCount=11, AvailableMemoryMB=2877 2024-11-13T18:25:41,458 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:41,460 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:41,489 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:41,491 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-60895553, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-60895553, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:41,540 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-60895553/hregion-60895553.1731522341492, exclude list is [], retry=0 2024-11-13T18:25:41,547 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:41,547 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:41,548 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:41,556 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-60895553/hregion-60895553.1731522341492 2024-11-13T18:25:41,561 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:41,561 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 1f8201c7c1e7b0cea225f11de7946e54, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:41,617 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741879_1056 (size=70) 2024-11-13T18:25:41,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741879_1056 (size=70) 2024-11-13T18:25:41,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741879_1056 (size=70) 2024-11-13T18:25:42,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:42,020 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,022 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName a 2024-11-13T18:25:42,022 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,023 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,023 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,024 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName b 2024-11-13T18:25:42,024 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,025 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,025 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,026 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName c 2024-11-13T18:25:42,026 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,026 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,027 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,028 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,028 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,029 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,029 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:42,030 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,032 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:42,033 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f8201c7c1e7b0cea225f11de7946e54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62136832, jitterRate=-0.07408905029296875}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:42,033 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f8201c7c1e7b0cea225f11de7946e54: Writing region info on filesystem at 1731522342019Initializing all the Stores at 1731522342020 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342020Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342020Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342020Cleaning up temporary data from old regions at 1731522342029 (+9 ms)Region opened successfully at 1731522342033 (+4 ms) 2024-11-13T18:25:42,034 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f8201c7c1e7b0cea225f11de7946e54, disabling compactions & flushes 2024-11-13T18:25:42,034 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:42,034 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:42,034 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. after waiting 0 ms 2024-11-13T18:25:42,034 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:42,034 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:42,034 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f8201c7c1e7b0cea225f11de7946e54: Waiting for close lock at 1731522342034Disabling compacts and flushes for region at 1731522342034Disabling writes for close at 1731522342034Writing region close event to WAL at 1731522342034Closed at 1731522342034 2024-11-13T18:25:42,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741878_1055 (size=93) 2024-11-13T18:25:42,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741878_1055 (size=93) 2024-11-13T18:25:42,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741878_1055 (size=93) 2024-11-13T18:25:42,041 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:42,041 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-60895553:(num 1731522341492) 2024-11-13T18:25:42,041 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:42,043 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:42,066 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, exclude list is [], retry=0 2024-11-13T18:25:42,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:42,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:42,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:42,072 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 2024-11-13T18:25:42,073 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:42,073 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f8201c7c1e7b0cea225f11de7946e54, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:42,073 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:42,074 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,074 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,076 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,077 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName a 2024-11-13T18:25:42,077 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,078 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,078 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,079 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName b 2024-11-13T18:25:42,079 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,079 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,079 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,080 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName c 2024-11-13T18:25:42,080 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:42,081 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:42,081 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,082 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,084 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,084 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,085 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:42,087 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:42,088 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f8201c7c1e7b0cea225f11de7946e54; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59566782, jitterRate=-0.11238577961921692}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:42,089 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f8201c7c1e7b0cea225f11de7946e54: Writing region info on filesystem at 1731522342074Initializing all the Stores at 1731522342075 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342075Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342076 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522342076Cleaning up temporary data from old regions at 1731522342084 (+8 ms)Region opened successfully at 1731522342089 (+5 ms) 2024-11-13T18:25:42,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731522342092/Put/seqid=0 2024-11-13T18:25:42,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741881_1058 (size=4826) 2024-11-13T18:25:42,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741881_1058 (size=4826) 2024-11-13T18:25:42,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741881_1058 (size=4826) 2024-11-13T18:25:42,102 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40367/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 1f8201c7c1e7b0cea225f11de7946e54/a 2024-11-13T18:25:42,112 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T18:25:42,112 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T18:25:42,112 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 1f8201c7c1e7b0cea225f11de7946e54: 2024-11-13T18:25:42,114 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_ 2024-11-13T18:25:42,115 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40367/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1f8201c7c1e7b0cea225f11de7946e54/a as hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_ - updating store file list. 2024-11-13T18:25:42,122 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T18:25:42,122 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_ into 1f8201c7c1e7b0cea225f11de7946e54/a 2024-11-13T18:25:42,122 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40367/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 1f8201c7c1e7b0cea225f11de7946e54/a (new location: hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_) 2024-11-13T18:25:42,167 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, size=0 (0bytes) 2024-11-13T18:25:42,167 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 might be still open, length is 0 2024-11-13T18:25:42,167 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 2024-11-13T18:25:42,167 WARN [IPC Server handler 1 on default port 40367 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T18:25:42,168 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 after 0ms 2024-11-13T18:25:44,228 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:43802 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:46817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43802 dst: /127.0.0.1:46817 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:46817 remote=/127.0.0.1:43802]. Total timeout mills is 60000, 57902 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:44,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:56218 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:33523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56218 dst: /127.0.0.1:33523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:44,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:35476 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35476 dst: /127.0.0.1:41041 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:44,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741880_1059 (size=504) 2024-11-13T18:25:44,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741880_1059 (size=504) 2024-11-13T18:25:46,168 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 after 4001ms 2024-11-13T18:25:46,172 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 took 4006ms 2024-11-13T18:25:46,174 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044; continuing. 2024-11-13T18:25:46,174 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 so closing down 2024-11-13T18:25:46,174 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:46,176 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731522342044.temp 2024-11-13T18:25:46,177 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp 2024-11-13T18:25:46,177 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741882_1060 (size=238) 2024-11-13T18:25:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741882_1060 (size=238) 2024-11-13T18:25:46,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741882_1060 (size=238) 2024-11-13T18:25:46,189 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:46,190 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp to hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 2024-11-13T18:25:46,190 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T18:25:46,190 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, journal: Splitting hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, size=0 (0bytes) at 1731522342167Finishing writing output for hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 so closing down at 1731522346174 (+4007 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp at 1731522346177 (+3 ms)3 split writer threads finished at 1731522346177Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731522346189 (+12 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005-wal.1731522342044.temp to hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 at 1731522346190 (+1 ms)Processed 2 edits across 1 Regions in 18 ms; skipped=1; WAL=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044, size=0, length=0, corrupted=false, cancelled=false at 1731522346190 2024-11-13T18:25:46,192 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522342044 2024-11-13T18:25:46,193 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 2024-11-13T18:25:46,193 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:46,195 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:46,208 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522346195, exclude list is [], retry=0 2024-11-13T18:25:46,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:46,210 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:46,211 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:46,215 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522346195 2024-11-13T18:25:46,215 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:25:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1f8201c7c1e7b0cea225f11de7946e54, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,216 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,219 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,221 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName a 2024-11-13T18:25:46,221 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,229 DEBUG [StoreFileOpener-1f8201c7c1e7b0cea225f11de7946e54-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T18:25:46,230 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/50df2bbd9a274e378f202efbe2ecfbc2_SeqId_3_ 2024-11-13T18:25:46,230 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:46,230 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,231 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName b 2024-11-13T18:25:46,231 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,232 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:46,232 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,233 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1f8201c7c1e7b0cea225f11de7946e54 columnFamilyName c 2024-11-13T18:25:46,233 DEBUG [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,234 INFO [StoreOpener-1f8201c7c1e7b0cea225f11de7946e54-1 {}] regionserver.HStore(327): Store=1f8201c7c1e7b0cea225f11de7946e54/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:46,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,237 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,238 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 2024-11-13T18:25:46,241 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 2024-11-13T18:25:46,241 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 1f8201c7c1e7b0cea225f11de7946e54 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T18:25:46,264 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/.tmp/a/7e36da07de2f464d8a1e4e1346eb325d is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731522342129/Put/seqid=0 2024-11-13T18:25:46,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741884_1062 (size=5149) 2024-11-13T18:25:46,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741884_1062 (size=5149) 2024-11-13T18:25:46,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741884_1062 (size=5149) 2024-11-13T18:25:46,273 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/.tmp/a/7e36da07de2f464d8a1e4e1346eb325d 2024-11-13T18:25:46,280 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/.tmp/a/7e36da07de2f464d8a1e4e1346eb325d as hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/7e36da07de2f464d8a1e4e1346eb325d 2024-11-13T18:25:46,287 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/a/7e36da07de2f464d8a1e4e1346eb325d, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T18:25:46,287 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1f8201c7c1e7b0cea225f11de7946e54 in 46ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T18:25:46,288 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/0000000000000000005 2024-11-13T18:25:46,289 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,289 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,290 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:46,292 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 1f8201c7c1e7b0cea225f11de7946e54 2024-11-13T18:25:46,294 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/1f8201c7c1e7b0cea225f11de7946e54/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T18:25:46,295 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 1f8201c7c1e7b0cea225f11de7946e54; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72256067, jitterRate=0.07669930160045624}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:46,295 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 1f8201c7c1e7b0cea225f11de7946e54: Writing region info on filesystem at 1731522346216Initializing all the Stores at 1731522346217 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522346217Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522346219 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522346219Obtaining lock to block concurrent updates at 1731522346241 (+22 ms)Preparing flush snapshotting stores in 1f8201c7c1e7b0cea225f11de7946e54 at 1731522346241Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731522346241Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. at 1731522346241Flushing 1f8201c7c1e7b0cea225f11de7946e54/a: creating writer at 1731522346241Flushing 1f8201c7c1e7b0cea225f11de7946e54/a: appending metadata at 1731522346263 (+22 ms)Flushing 1f8201c7c1e7b0cea225f11de7946e54/a: closing flushed file at 1731522346263Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cc119d3: reopening flushed file at 1731522346279 (+16 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 1f8201c7c1e7b0cea225f11de7946e54 in 46ms, sequenceid=5, compaction requested=false; wal=null at 1731522346287 (+8 ms)Cleaning up temporary data from old regions at 1731522346289 (+2 ms)Region opened successfully at 1731522346295 (+6 ms) 2024-11-13T18:25:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 1f8201c7c1e7b0cea225f11de7946e54, disabling compactions & flushes 2024-11-13T18:25:46,299 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. after waiting 0 ms 2024-11-13T18:25:46,299 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:46,301 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731522341459.1f8201c7c1e7b0cea225f11de7946e54. 2024-11-13T18:25:46,301 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 1f8201c7c1e7b0cea225f11de7946e54: Waiting for close lock at 1731522346299Disabling compacts and flushes for region at 1731522346299Disabling writes for close at 1731522346299Writing region close event to WAL at 1731522346300 (+1 ms)Closed at 1731522346300 2024-11-13T18:25:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741883_1061 (size=93) 2024-11-13T18:25:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741883_1061 (size=93) 2024-11-13T18:25:46,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741883_1061 (size=93) 2024-11-13T18:25:46,307 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:46,307 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731522346195) 2024-11-13T18:25:46,324 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=410 (was 405) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40367 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:40367 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1481209767_22 at /127.0.0.1:48474 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1481209767_22 at /127.0.0.1:58412 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1024 (was 961) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=521 (was 514) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2659 (was 2877) 2024-11-13T18:25:46,339 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410, OpenFileDescriptor=1024, MaxFileDescriptor=1048576, SystemLoadAverage=521, ProcessCount=11, AvailableMemoryMB=2659 2024-11-13T18:25:46,354 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:46,359 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T18:25:46,363 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fda96cf025dc,42825,1731522327393 2024-11-13T18:25:46,366 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@38924da9 2024-11-13T18:25:46,367 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T18:25:46,369 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:58850, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T18:25:46,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T18:25:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T18:25:46,380 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T18:25:46,382 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T18:25:46,382 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,384 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T18:25:46,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:25:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741885_1063 (size=694) 2024-11-13T18:25:46,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741885_1063 (size=694) 2024-11-13T18:25:46,395 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741885_1063 (size=694) 2024-11-13T18:25:46,397 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4 2024-11-13T18:25:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741886_1064 (size=77) 2024-11-13T18:25:46,404 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741886_1064 (size=77) 2024-11-13T18:25:46,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741886_1064 (size=77) 2024-11-13T18:25:46,405 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:46,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:46,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:46,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,406 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,406 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522346406Disabling compacts and flushes for region at 1731522346406Disabling writes for close at 1731522346406Writing region close event to WAL at 1731522346406Closed at 1731522346406 2024-11-13T18:25:46,407 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T18:25:46,412 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731522346407"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731522346407"}]},"ts":"1731522346407"} 2024-11-13T18:25:46,419 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T18:25:46,420 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T18:25:46,423 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522346420"}]},"ts":"1731522346420"} 2024-11-13T18:25:46,426 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T18:25:46,427 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T18:25:46,428 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:25:46,428 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:25:46,428 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T18:25:46,428 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:25:46,429 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN}] 2024-11-13T18:25:46,431 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN 2024-11-13T18:25:46,433 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN; state=OFFLINE, location=fda96cf025dc,41739,1731522328489; forceNewPlan=false, retain=false 2024-11-13T18:25:46,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:25:46,585 INFO [fda96cf025dc:42825 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:25:46,586 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPENING, regionLocation=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:46,589 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN because future has completed 2024-11-13T18:25:46,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489}] 2024-11-13T18:25:46,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:25:46,744 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:25:46,746 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48849, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:25:46,752 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,752 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:46,752 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,752 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:46,752 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,752 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,754 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,756 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:46,756 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,757 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:46,757 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,759 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:46,759 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:46,759 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:46,759 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,760 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,761 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,762 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,762 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,763 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T18:25:46,765 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,767 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:46,768 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 99991ec418f4188651e0104287d02355; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70079736, jitterRate=0.044269442558288574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T18:25:46,768 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:46,769 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 99991ec418f4188651e0104287d02355: Running coprocessor pre-open hook at 1731522346753Writing region info on filesystem at 1731522346753Initializing all the Stores at 1731522346754 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522346754Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522346754Cleaning up temporary data from old regions at 1731522346762 (+8 ms)Running coprocessor post-open hooks at 1731522346768 (+6 ms)Region opened successfully at 1731522346769 (+1 ms) 2024-11-13T18:25:46,771 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., pid=6, masterSystemTime=1731522346744 2024-11-13T18:25:46,774 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,774 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:46,775 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPEN, openSeqNum=2, regionLocation=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:46,780 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 because future has completed 2024-11-13T18:25:46,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T18:25:46,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 in 191 msec 2024-11-13T18:25:46,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T18:25:46,789 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN in 356 msec 2024-11-13T18:25:46,790 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T18:25:46,790 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731522346790"}]},"ts":"1731522346790"} 2024-11-13T18:25:46,793 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T18:25:46,794 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T18:25:46,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 420 msec 2024-11-13T18:25:47,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T18:25:47,020 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T18:25:47,020 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T18:25:47,022 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:25:47,028 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T18:25:47,028 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T18:25:47,029 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T18:25:47,043 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=2] 2024-11-13T18:25:47,045 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:47,047 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51064, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:47,062 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=99991ec418f4188651e0104287d02355, source=fda96cf025dc,41739,1731522328489, destination=fda96cf025dc,33531,1731522328314, warming up region on fda96cf025dc,33531,1731522328314 2024-11-13T18:25:47,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:25:47,065 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=99991ec418f4188651e0104287d02355, source=fda96cf025dc,41739,1731522328489, destination=fda96cf025dc,33531,1731522328314, running balancer 2024-11-13T18:25:47,066 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55499, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:25:47,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE 2024-11-13T18:25:47,066 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE 2024-11-13T18:25:47,068 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=CLOSING, regionLocation=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:47,071 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(7855): Warmup {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:47,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:47,072 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,072 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE because future has completed 2024-11-13T18:25:47,073 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:47,073 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:47,074 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:47,074 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,075 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:47,075 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:47,075 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:47,076 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,080 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:25:47,080 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489}] 2024-11-13T18:25:47,080 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522347076Disabling compacts and flushes for region at 1731522347076Disabling writes for close at 1731522347076Writing region close event to WAL at 1731522347080 (+4 ms)Closed at 1731522347080 2024-11-13T18:25:47,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T18:25:47,239 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,239 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:25:47,240 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:47,240 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,240 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,240 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:47,240 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,240 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 99991ec418f4188651e0104287d02355 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T18:25:47,259 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/7f458046f02b43c78373bacd9e1684d5 is 35, key is r1/cf1:q/1731522347047/Put/seqid=0 2024-11-13T18:25:47,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741887_1065 (size=4783) 2024-11-13T18:25:47,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741887_1065 (size=4783) 2024-11-13T18:25:47,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741887_1065 (size=4783) 2024-11-13T18:25:47,272 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/7f458046f02b43c78373bacd9e1684d5 2024-11-13T18:25:47,281 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/7f458046f02b43c78373bacd9e1684d5 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5 2024-11-13T18:25:47,289 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T18:25:47,291 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 99991ec418f4188651e0104287d02355 in 51ms, sequenceid=5, compaction requested=false 2024-11-13T18:25:47,291 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T18:25:47,301 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:25:47,305 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,305 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522347240Running coprocessor pre-close hooks at 1731522347240Disabling compacts and flushes for region at 1731522347240Disabling writes for close at 1731522347240Obtaining lock to block concurrent updates at 1731522347240Preparing flush snapshotting stores in 99991ec418f4188651e0104287d02355 at 1731522347240Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731522347241 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. at 1731522347242 (+1 ms)Flushing 99991ec418f4188651e0104287d02355/cf1: creating writer at 1731522347242Flushing 99991ec418f4188651e0104287d02355/cf1: appending metadata at 1731522347259 (+17 ms)Flushing 99991ec418f4188651e0104287d02355/cf1: closing flushed file at 1731522347259Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12fe5e2d: reopening flushed file at 1731522347280 (+21 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 99991ec418f4188651e0104287d02355 in 51ms, sequenceid=5, compaction requested=false at 1731522347291 (+11 ms)Writing region close event to WAL at 1731522347297 (+6 ms)Running coprocessor post-close hooks at 1731522347302 (+5 ms)Closed at 1731522347305 (+3 ms) 2024-11-13T18:25:47,306 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 99991ec418f4188651e0104287d02355 move to fda96cf025dc,33531,1731522328314 record at close sequenceid=5 2024-11-13T18:25:47,310 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,311 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=CLOSED 2024-11-13T18:25:47,314 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 because future has completed 2024-11-13T18:25:47,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T18:25:47,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 in 237 msec 2024-11-13T18:25:47,330 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,33531,1731522328314; forceNewPlan=false, retain=false 2024-11-13T18:25:47,481 INFO [fda96cf025dc:42825 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:25:47,481 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPENING, regionLocation=fda96cf025dc,33531,1731522328314 2024-11-13T18:25:47,484 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE because future has completed 2024-11-13T18:25:47,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314}] 2024-11-13T18:25:47,642 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,642 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:47,642 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,642 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:47,643 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,643 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,645 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,646 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:47,646 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:47,654 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5 2024-11-13T18:25:47,654 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:47,654 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,656 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:47,656 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:47,656 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:47,656 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,657 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,659 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,660 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,660 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,661 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T18:25:47,663 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,664 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 99991ec418f4188651e0104287d02355; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69220506, jitterRate=0.03146591782569885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T18:25:47,664 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:47,665 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 99991ec418f4188651e0104287d02355: Running coprocessor pre-open hook at 1731522347643Writing region info on filesystem at 1731522347643Initializing all the Stores at 1731522347644 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522347644Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522347644Cleaning up temporary data from old regions at 1731522347660 (+16 ms)Running coprocessor post-open hooks at 1731522347664 (+4 ms)Region opened successfully at 1731522347665 (+1 ms) 2024-11-13T18:25:47,669 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., pid=9, masterSystemTime=1731522347637 2024-11-13T18:25:47,671 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,671 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:47,673 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPEN, openSeqNum=9, regionLocation=fda96cf025dc,33531,1731522328314 2024-11-13T18:25:47,675 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314 because future has completed 2024-11-13T18:25:47,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T18:25:47,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314 in 193 msec 2024-11-13T18:25:47,682 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE in 615 msec 2024-11-13T18:25:47,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:25:47,726 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:25:47,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:51064 deadline: 1731522407730, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=33531 startCode=1731522328314. As of locationSeqNum=5. 2024-11-13T18:25:47,761 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=33531 startCode=1731522328314. As of locationSeqNum=5. 2024-11-13T18:25:47,762 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=33531 startCode=1731522328314. As of locationSeqNum=5. 2024-11-13T18:25:47,762 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,33531,1731522328314, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=33531 startCode=1731522328314. As of locationSeqNum=5. 2024-11-13T18:25:47,869 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T18:25:47,871 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43444, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T18:25:47,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T18:25:47,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:47,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T18:25:47,880 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:47,881 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T18:25:47,881 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:47,881 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 99991ec418f4188651e0104287d02355 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T18:25:47,898 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/496ab23926e34a23aeb1c2e2d2e60470 is 29, key is r1/cf1:/1731522347872/DeleteFamily/seqid=0 2024-11-13T18:25:47,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741888_1066 (size=4906) 2024-11-13T18:25:47,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741888_1066 (size=4906) 2024-11-13T18:25:47,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741888_1066 (size=4906) 2024-11-13T18:25:48,312 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,319 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,338 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 is 29, key is r1/cf2:/1731522347872/DeleteFamily/seqid=0 2024-11-13T18:25:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741889_1067 (size=4906) 2024-11-13T18:25:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741889_1067 (size=4906) 2024-11-13T18:25:48,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741889_1067 (size=4906) 2024-11-13T18:25:48,346 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,352 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,353 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/496ab23926e34a23aeb1c2e2d2e60470 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,361 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,361 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T18:25:48,362 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,369 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,369 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T18:25:48,370 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 99991ec418f4188651e0104287d02355 in 489ms, sequenceid=12, compaction requested=false 2024-11-13T18:25:48,370 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 99991ec418f4188651e0104287d02355: 2024-11-13T18:25:48,373 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T18:25:48,374 DEBUG [Time-limited test {}] regionserver.HStore(1541): 99991ec418f4188651e0104287d02355/cf1 is initiating major compaction (all files) 2024-11-13T18:25:48,374 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:25:48,375 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:48,375 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 99991ec418f4188651e0104287d02355/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,375 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5, hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470] into tmpdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp, totalSize=9.5 K 2024-11-13T18:25:48,376 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7f458046f02b43c78373bacd9e1684d5, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731522347047 2024-11-13T18:25:48,377 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 496ab23926e34a23aeb1c2e2d2e60470, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T18:25:48,390 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 99991ec418f4188651e0104287d02355#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:48,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741890_1068 (size=4626) 2024-11-13T18:25:48,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741890_1068 (size=4626) 2024-11-13T18:25:48,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741890_1068 (size=4626) 2024-11-13T18:25:48,407 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf1/87377bed51a34c9391950e93eb3186e2 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/87377bed51a34c9391950e93eb3186e2 2024-11-13T18:25:48,425 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 99991ec418f4188651e0104287d02355/cf1 of 99991ec418f4188651e0104287d02355 into 87377bed51a34c9391950e93eb3186e2(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:48,425 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 99991ec418f4188651e0104287d02355: 2024-11-13T18:25:48,425 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T18:25:48,425 DEBUG [Time-limited test {}] regionserver.HStore(1541): 99991ec418f4188651e0104287d02355/cf2 is initiating major compaction (all files) 2024-11-13T18:25:48,426 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T18:25:48,426 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T18:25:48,426 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 99991ec418f4188651e0104287d02355/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,426 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5] into tmpdir=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp, totalSize=4.8 K 2024-11-13T18:25:48,427 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting feab2e45821c4fb3bd9dd3b3555d7fe5, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T18:25:48,437 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 99991ec418f4188651e0104287d02355#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T18:25:48,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741891_1069 (size=4592) 2024-11-13T18:25:48,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741891_1069 (size=4592) 2024-11-13T18:25:48,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741891_1069 (size=4592) 2024-11-13T18:25:48,465 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/.tmp/cf2/97960e6afb0a4ae9981a9b4a51337260 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/97960e6afb0a4ae9981a9b4a51337260 2024-11-13T18:25:48,474 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 99991ec418f4188651e0104287d02355/cf2 of 99991ec418f4188651e0104287d02355 into 97960e6afb0a4ae9981a9b4a51337260(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:25:48,474 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 99991ec418f4188651e0104287d02355: 2024-11-13T18:25:48,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=99991ec418f4188651e0104287d02355, source=fda96cf025dc,33531,1731522328314, destination=fda96cf025dc,41739,1731522328489, warming up region on fda96cf025dc,41739,1731522328489 2024-11-13T18:25:48,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=99991ec418f4188651e0104287d02355, source=fda96cf025dc,33531,1731522328314, destination=fda96cf025dc,41739,1731522328489, running balancer 2024-11-13T18:25:48,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE 2024-11-13T18:25:48,482 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE 2024-11-13T18:25:48,483 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(7855): Warmup {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:48,484 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=CLOSING, regionLocation=fda96cf025dc,33531,1731522328314 2024-11-13T18:25:48,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:48,484 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:48,486 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE because future has completed 2024-11-13T18:25:48,487 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T18:25:48,487 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314}] 2024-11-13T18:25:48,489 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:48,489 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:48,499 INFO [StoreFileOpener-99991ec418f4188651e0104287d02355-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,499 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,504 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5 2024-11-13T18:25:48,508 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/87377bed51a34c9391950e93eb3186e2 2024-11-13T18:25:48,508 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:48,509 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:48,510 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:48,510 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:48,519 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/97960e6afb0a4ae9981a9b4a51337260 2024-11-13T18:25:48,524 INFO [StoreFileOpener-99991ec418f4188651e0104287d02355-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,525 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,525 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:48,525 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:48,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,529 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41739 {}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522348525Disabling compacts and flushes for region at 1731522348525Disabling writes for close at 1731522348525Writing region close event to WAL at 1731522348529 (+4 ms)Closed at 1731522348529 2024-11-13T18:25:48,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T18:25:48,643 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 99991ec418f4188651e0104287d02355 2024-11-13T18:25:48,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T18:25:48,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:48,643 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:48,643 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,645 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5, hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470] to archive 2024-11-13T18:25:48,649 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:48,654 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5 to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/7f458046f02b43c78373bacd9e1684d5 2024-11-13T18:25:48,659 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470 to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/496ab23926e34a23aeb1c2e2d2e60470 2024-11-13T18:25:48,678 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5] to archive 2024-11-13T18:25:48,679 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:25:48,682 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/feab2e45821c4fb3bd9dd3b3555d7fe5 2024-11-13T18:25:48,690 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T18:25:48,690 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:48,690 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522348643Running coprocessor pre-close hooks at 1731522348643Disabling compacts and flushes for region at 1731522348643Disabling writes for close at 1731522348643Writing region close event to WAL at 1731522348685 (+42 ms)Running coprocessor post-close hooks at 1731522348690 (+5 ms)Closed at 1731522348690 2024-11-13T18:25:48,691 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 99991ec418f4188651e0104287d02355 move to fda96cf025dc,41739,1731522328489 record at close sequenceid=12 2024-11-13T18:25:48,695 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 99991ec418f4188651e0104287d02355 2024-11-13T18:25:48,698 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=CLOSED 2024-11-13T18:25:48,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314 because future has completed 2024-11-13T18:25:48,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T18:25:48,710 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,33531,1731522328314 in 219 msec 2024-11-13T18:25:48,711 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE; state=CLOSED, location=fda96cf025dc,41739,1731522328489; forceNewPlan=false, retain=false 2024-11-13T18:25:48,861 INFO [fda96cf025dc:42825 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T18:25:48,862 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPENING, regionLocation=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:48,866 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE because future has completed 2024-11-13T18:25:48,867 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489}] 2024-11-13T18:25:49,026 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,026 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:49,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:49,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,027 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,029 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,030 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:49,030 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:49,038 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/87377bed51a34c9391950e93eb3186e2 2024-11-13T18:25:49,038 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:49,038 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,039 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:49,039 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:49,045 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/97960e6afb0a4ae9981a9b4a51337260 2024-11-13T18:25:49,045 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:49,045 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,046 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,048 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,048 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,049 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,049 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T18:25:49,051 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,051 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 99991ec418f4188651e0104287d02355; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63172565, jitterRate=-0.058655425906181335}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T18:25:49,052 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,052 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 99991ec418f4188651e0104287d02355: Running coprocessor pre-open hook at 1731522349027Writing region info on filesystem at 1731522349027Initializing all the Stores at 1731522349028 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522349028Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522349029 (+1 ms)Cleaning up temporary data from old regions at 1731522349049 (+20 ms)Running coprocessor post-open hooks at 1731522349052 (+3 ms)Region opened successfully at 1731522349052 2024-11-13T18:25:49,053 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., pid=12, masterSystemTime=1731522349021 2024-11-13T18:25:49,056 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,056 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,057 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPEN, openSeqNum=18, regionLocation=fda96cf025dc,41739,1731522328489 2024-11-13T18:25:49,060 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 because future has completed 2024-11-13T18:25:49,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T18:25:49,064 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,41739,1731522328489 in 194 msec 2024-11-13T18:25:49,069 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, REOPEN/MOVE in 584 msec 2024-11-13T18:25:49,084 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T18:25:49,085 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51066, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T18:25:49,087 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server fda96cf025dc,41739,1731522328489: testing ***** 2024-11-13T18:25:49,087 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T18:25:49,089 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T18:25:49,090 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T18:25:49,096 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T18:25:49,098 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T18:25:49,111 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 329532928 }, "NonHeapMemoryUsage": { "committed": 171311104, "init": 7667712, "max": -1, "used": 168679888 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 11067, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 49, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 4, "ProcessCallTime_90th_percentile": 5, "ProcessCallTime_95th_percentile": 14, "ProcessCallTime_98th_percentile": 49, "ProcessCallTime_99th_percentile": 49, "ProcessCallTime_99.9th_percentile": 49, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 14, "ProcessCallTime_TimeRangeCount_3-10": 6, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 1, "QueueCallTime_99th_percentile": 1, "QueueCallTime_99.9th_percentile": 1, "QueueCallTime_TimeRangeCount_0-1": 31, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 49, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 4, "TotalCallTime_90th_percentile": 5, "TotalCallTime_95th_percentile": 14, "TotalCallTime_98th_percentile": 49, "TotalCallTime_99th_percentile": 49, "TotalCallTime_99.9th_percentile": 49, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 14, "TotalCallTime_TimeRangeCount_3-10": 6, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 158, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 5, "ResponseSize_SizeRangeCount_100-1000": 4, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5743 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:54860", "tag.serverName": "fda96cf025dc,41739,1731522328489", "tag.clusterId": "bd0e7932-69d1-4437-b0c5-be1e6f1fb465", "tag.Context": "regionserver", "tag.Hostname": "fda96cf025dc", "regionCount": 1, "storeCount": 0, "hlogFileCount": 3, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731522328489, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227243, "ByteBuffAllocatorPoolAllocationBytes": 66560, "ByteBuffAllocatorHeapAllocationRatio": 0.7697053155307054, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 4, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 1, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 2, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 1, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "ScanTime_TimeRangeCount_0-1": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 1, "Put_min": 3, "Put_max": 3, "Put_mean": 3, "Put_25th_percentile": 3, "Put_median": 3, "Put_75th_percentile": 3, "Put_90th_percentile": 3, "Put_95th_percentile": 3, "Put_98th_percentile": 3, "Put_99th_percentile": 3, "Put_99.9th_percentile": 3, "Put_TimeRangeCount_1-3": 1, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 4783, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 31, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 4783, "FlushOutputSize_max": 4783, "FlushOutputSize_mean": 4783, "FlushOutputSize_25th_percentile": 4783, "FlushOutputSize_median": 4783, "FlushOutputSize_75th_percentile": 4783, "FlushOutputSize_90th_percentile": 4783, "FlushOutputSize_95th_percentile": 4783, "FlushOutputSize_98th_percentile": 4783, "FlushOutputSize_99th_percentile": 4783, "FlushOutputSize_99.9th_percentile": 4783, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 31, "FlushMemstoreSize_max": 31, "FlushMemstoreSize_mean": 31, "FlushMemstoreSize_25th_percentile": 31, "FlushMemstoreSize_median": 31, "FlushMemstoreSize_75th_percentile": 31, "FlushMemstoreSize_90th_percentile": 31, "FlushMemstoreSize_95th_percentile": 31, "FlushMemstoreSize_98th_percentile": 31, "FlushMemstoreSize_99th_percentile": 31, "FlushMemstoreSize_99.9th_percentile": 31, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 51, "FlushTime_max": 51, "FlushTime_mean": 51, "FlushTime_25th_percentile": 51, "FlushTime_median": 51, "FlushTime_75th_percentile": 51, "FlushTime_90th_percentile": 51, "FlushTime_95th_percentile": 51, "FlushTime_98th_percentile": 51, "FlushTime_99th_percentile": 51, "FlushTime_99.9th_percentile": 51, "FlushTime_TimeRangeCount_30-100": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 1, "ScanSize_min": 64, "ScanSize_max": 64, "ScanSize_mean": 64, "ScanSize_25th_percentile": 64, "ScanSize_median": 64, "ScanSize_75th_percentile": 64, "ScanSize_90th_percentile": 64, "ScanSize_95th_percentile": 64, "ScanSize_98th_percentile": 64, "ScanSize_99th_percentile": 64, "ScanSize_99.9th_percentile": 64, "ScanSize_SizeRangeCount_10-100": 1, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T18:25:49,115 WARN [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42825 {}] master.MasterRpcServices(700): fda96cf025dc,41739,1731522328489 reported a fatal error: ***** ABORTING region server fda96cf025dc,41739,1731522328489: testing ***** 2024-11-13T18:25:49,118 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,41739,1731522328489' ***** 2024-11-13T18:25:49,118 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T18:25:49,118 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:25:49,119 INFO [RS:2;fda96cf025dc:41739 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T18:25:49,119 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:25:49,119 INFO [RS:2;fda96cf025dc:41739 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T18:25:49,119 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(3091): Received CLOSE for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33531 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:43444 deadline: 1731522409119, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=41739 startCode=1731522328489. As of locationSeqNum=12. 2024-11-13T18:25:49,119 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(956): aborting server fda96cf025dc,41739,1731522328489 2024-11-13T18:25:49,120 INFO [RS:2;fda96cf025dc:41739 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:25:49,120 INFO [RS:2;fda96cf025dc:41739 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;fda96cf025dc:41739. 2024-11-13T18:25:49,120 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:25:49,120 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,120 DEBUG [RS:2;fda96cf025dc:41739 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:25:49,120 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,120 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:25:49,120 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,120 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,33531,1731522328314, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,33531,1731522328314, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=41739 startCode=1731522328489. As of locationSeqNum=12. 2024-11-13T18:25:49,121 DEBUG [RS:2;fda96cf025dc:41739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:49,121 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,33531,1731522328314, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=41739 startCode=1731522328489. As of locationSeqNum=12. 2024-11-13T18:25:49,121 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,33531,1731522328314, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=fda96cf025dc port=41739 startCode=1731522328489. As of locationSeqNum=12. 2024-11-13T18:25:49,121 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T18:25:49,121 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1325): Online Regions={99991ec418f4188651e0104287d02355=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.} 2024-11-13T18:25:49,121 DEBUG [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1351): Waiting on 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,123 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,123 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522349119Running coprocessor pre-close hooks at 1731522349120 (+1 ms)Disabling compacts and flushes for region at 1731522349120Disabling writes for close at 1731522349120Writing region close event to WAL at 1731522349123 (+3 ms)Running coprocessor post-close hooks at 1731522349123Closed at 1731522349123 2024-11-13T18:25:49,124 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,145 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:25:49,229 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fda96cf025dc,41739,1731522328489 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,230 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fda96cf025dc,41739,1731522328489 aborting 2024-11-13T18:25:49,231 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server fda96cf025dc,41739,1731522328489 aborting 2024-11-13T18:25:49,231 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=12 from cache 2024-11-13T18:25:49,322 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,41739,1731522328489; all regions closed. 2024-11-13T18:25:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741834_1010 (size=2142) 2024-11-13T18:25:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741834_1010 (size=2142) 2024-11-13T18:25:49,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741834_1010 (size=2142) 2024-11-13T18:25:49,326 DEBUG [RS:2;fda96cf025dc:41739 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:25:49,326 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:25:49,326 INFO [RS:2;fda96cf025dc:41739 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:25:49,327 INFO [RS:2;fda96cf025dc:41739 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:25:49,327 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:25:49,327 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:25:49,327 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:25:49,327 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:25:49,327 INFO [RS:2;fda96cf025dc:41739 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:25:49,328 INFO [RS:2;fda96cf025dc:41739 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41739 2024-11-13T18:25:49,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,41739,1731522328489 2024-11-13T18:25:49,336 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:25:49,336 INFO [RS:2;fda96cf025dc:41739 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:25:49,337 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,41739,1731522328489] 2024-11-13T18:25:49,340 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,41739,1731522328489 already deleted, retry=false 2024-11-13T18:25:49,340 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of fda96cf025dc,41739,1731522328489 on fda96cf025dc,42825,1731522327393 2024-11-13T18:25:49,345 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure fda96cf025dc,41739,1731522328489, splitWal=true, meta=false 2024-11-13T18:25:49,347 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure fda96cf025dc,41739,1731522328489, splitWal=true, meta=false 2024-11-13T18:25:49,347 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for fda96cf025dc,41739,1731522328489 (carryingMeta=false) fda96cf025dc,41739,1731522328489/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@79d5c1a4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T18:25:49,349 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(207): fda96cf025dc,41739,1731522328489 had 1 regions 2024-11-13T18:25:49,350 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure fda96cf025dc,41739,1731522328489, splitWal=true, meta=false, isMeta: false 2024-11-13T18:25:49,351 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting 2024-11-13T18:25:49,353 INFO [PEWorker-2 {}] master.SplitWALManager(105): fda96cf025dc,41739,1731522328489 WAL count=1, meta=false 2024-11-13T18:25:49,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure fda96cf025dc%2C41739%2C1731522328489.1731522330187}] 2024-11-13T18:25:49,359 DEBUG [PEWorker-4 {}] master.SplitWALManager(158): Acquired split WAL worker=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:49,361 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C41739%2C1731522328489.1731522330187, worker=fda96cf025dc,44035,1731522328435}] 2024-11-13T18:25:49,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:25:49,439 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41739-0x1003a6402a40003, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:25:49,440 INFO [RS:2;fda96cf025dc:41739 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:25:49,440 INFO [RS:2;fda96cf025dc:41739 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,41739,1731522328489; zookeeper connection closed. 2024-11-13T18:25:49,440 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6a27bad8 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6a27bad8 2024-11-13T18:25:49,443 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18] 2024-11-13T18:25:49,445 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server fda96cf025dc:41739 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:41739 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:49,446 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18, error=java.net.ConnectException: Call to address=fda96cf025dc:41739 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:41739 2024-11-13T18:25:49,446 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 is java.net.ConnectException: Connection refused 2024-11-13T18:25:49,446 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 from cache 2024-11-13T18:25:49,446 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address fda96cf025dc:41739 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: fda96cf025dc/172.17.0.3:41739 2024-11-13T18:25:49,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=44035 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T18:25:49,542 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187, size=2.1 K (2142bytes) 2024-11-13T18:25:49,542 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 2024-11-13T18:25:49,543 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 after 1ms 2024-11-13T18:25:49,546 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 took 4ms 2024-11-13T18:25:49,553 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 99991ec418f4188651e0104287d02355: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T18:25:49,553 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 so closing down 2024-11-13T18:25:49,553 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:49,554 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:49,554 INFO [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-13T18:25:49,554 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187, journal: Splitting hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187, size=2.1 K (2142bytes) at 1731522349542Finishing writing output for hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 so closing down at 1731522349553 (+11 ms)3 split writer threads finished at 1731522349554 (+1 ms)Processed 6 edits across 0 Regions in 8 ms; skipped=6; WAL=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731522349554 2024-11-13T18:25:49,554 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 2024-11-13T18:25:49,556 DEBUG [RS_LOG_REPLAY_OPS-regionserver/fda96cf025dc:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T18:25:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42825 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T18:25:49,562 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting/fda96cf025dc%2C41739%2C1731522328489.1731522330187 to hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs 2024-11-13T18:25:49,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T18:25:49,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure fda96cf025dc%2C41739%2C1731522328489.1731522330187, worker=fda96cf025dc,44035,1731522328435 in 201 msec 2024-11-13T18:25:49,566 DEBUG [PEWorker-3 {}] master.SplitWALManager(172): Release split WAL worker=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:49,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T18:25:49,569 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure fda96cf025dc%2C41739%2C1731522328489.1731522330187, worker=fda96cf025dc,44035,1731522328435 in 212 msec 2024-11-13T18:25:49,570 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting dir is empty, no logs to split. 2024-11-13T18:25:49,570 INFO [PEWorker-2 {}] master.SplitWALManager(105): fda96cf025dc,41739,1731522328489 WAL count=0, meta=false 2024-11-13T18:25:49,570 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if fda96cf025dc,41739,1731522328489 WAL splitting is done? wals=0, meta=false 2024-11-13T18:25:49,572 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for fda96cf025dc,41739,1731522328489 failed, ignore...File hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,41739,1731522328489-splitting does not exist. 2024-11-13T18:25:49,574 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN}] 2024-11-13T18:25:49,575 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN 2024-11-13T18:25:49,576 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(204): Hosts are {fda96cf025dc=0} racks are {/default-rack=0} 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T18:25:49,727 INFO [fda96cf025dc:42825 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T18:25:49,727 INFO [fda96cf025dc:42825 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T18:25:49,727 DEBUG [fda96cf025dc:42825 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T18:25:49,728 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPENING, regionLocation=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:49,730 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN because future has completed 2024-11-13T18:25:49,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,44035,1731522328435}] 2024-11-13T18:25:49,752 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18] 2024-11-13T18:25:49,753 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to fda96cf025dc:41739 this server is in the failed servers list 2024-11-13T18:25:49,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=fda96cf025dc:41739 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:41739 2024-11-13T18:25:49,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: fda96cf025dc:41739 2024-11-13T18:25:49,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,41739,1731522328489, seqNum=18 from cache 2024-11-13T18:25:49,888 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,888 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 99991ec418f4188651e0104287d02355, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:49,889 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,889 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:49,889 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,889 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,891 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,892 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf1 2024-11-13T18:25:49,892 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:49,902 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf1/87377bed51a34c9391950e93eb3186e2 2024-11-13T18:25:49,902 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:49,902 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,903 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 99991ec418f4188651e0104287d02355 columnFamilyName cf2 2024-11-13T18:25:49,903 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:49,909 DEBUG [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/cf2/97960e6afb0a4ae9981a9b4a51337260 2024-11-13T18:25:49,909 INFO [StoreOpener-99991ec418f4188651e0104287d02355-1 {}] regionserver.HStore(327): Store=99991ec418f4188651e0104287d02355/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:49,910 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,910 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,911 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,912 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,912 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,913 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T18:25:49,914 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,915 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 99991ec418f4188651e0104287d02355; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73659003, jitterRate=0.0976046770811081}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T18:25:49,915 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 99991ec418f4188651e0104287d02355 2024-11-13T18:25:49,915 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 99991ec418f4188651e0104287d02355: Running coprocessor pre-open hook at 1731522349889Writing region info on filesystem at 1731522349889Initializing all the Stores at 1731522349890 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522349891 (+1 ms)Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522349891Cleaning up temporary data from old regions at 1731522349912 (+21 ms)Running coprocessor post-open hooks at 1731522349915 (+3 ms)Region opened successfully at 1731522349915 2024-11-13T18:25:49,916 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., pid=17, masterSystemTime=1731522349883 2024-11-13T18:25:49,919 DEBUG [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,919 INFO [RS_OPEN_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:25:49,920 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=99991ec418f4188651e0104287d02355, regionState=OPEN, openSeqNum=18, regionLocation=fda96cf025dc,44035,1731522328435 2024-11-13T18:25:49,923 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,44035,1731522328435 because future has completed 2024-11-13T18:25:49,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T18:25:49,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 99991ec418f4188651e0104287d02355, server=fda96cf025dc,44035,1731522328435 in 193 msec 2024-11-13T18:25:49,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T18:25:49,928 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=99991ec418f4188651e0104287d02355, ASSIGN in 352 msec 2024-11-13T18:25:49,928 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server fda96cf025dc,41739,1731522328489 after splitting done 2024-11-13T18:25:49,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure fda96cf025dc,41739,1731522328489, splitWal=true, meta=false in 587 msec 2024-11-13T18:25:50,273 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., hostname=fda96cf025dc,44035,1731522328435, seqNum=18] 2024-11-13T18:25:50,291 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 410), OpenFileDescriptor=1051 (was 1024) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=536 (was 521) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2493 (was 2659) 2024-11-13T18:25:50,293 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T18:25:50,310 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1051, MaxFileDescriptor=1048576, SystemLoadAverage=536, ProcessCount=11, AvailableMemoryMB=2492 2024-11-13T18:25:50,310 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1051 is superior to 1024 2024-11-13T18:25:50,325 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:50,327 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:50,328 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:50,330 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43925281, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-43925281, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:50,343 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43925281/hregion-43925281.1731522350331, exclude list is [], retry=0 2024-11-13T18:25:50,346 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:50,347 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:50,347 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:50,350 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43925281/hregion-43925281.1731522350331 2024-11-13T18:25:50,351 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:50,351 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 93136835b15ffd96067339b6dd8e3691, NAME => 'testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741893_1071 (size=67) 2024-11-13T18:25:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741893_1071 (size=67) 2024-11-13T18:25:50,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741893_1071 (size=67) 2024-11-13T18:25:50,360 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:50,362 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,363 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName a 2024-11-13T18:25:50,363 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,364 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,364 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,365 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName b 2024-11-13T18:25:50,365 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,366 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,366 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,367 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName c 2024-11-13T18:25:50,367 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,368 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,368 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,368 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,369 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,370 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,370 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,370 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:50,372 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,374 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:50,374 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93136835b15ffd96067339b6dd8e3691; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66736173, jitterRate=-0.005553528666496277}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:50,375 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93136835b15ffd96067339b6dd8e3691: Writing region info on filesystem at 1731522350360Initializing all the Stores at 1731522350361 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350361Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350362 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350362Cleaning up temporary data from old regions at 1731522350370 (+8 ms)Region opened successfully at 1731522350375 (+5 ms) 2024-11-13T18:25:50,375 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 93136835b15ffd96067339b6dd8e3691, disabling compactions & flushes 2024-11-13T18:25:50,375 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,375 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,376 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. after waiting 0 ms 2024-11-13T18:25:50,376 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,376 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,376 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 93136835b15ffd96067339b6dd8e3691: Waiting for close lock at 1731522350375Disabling compacts and flushes for region at 1731522350375Disabling writes for close at 1731522350376 (+1 ms)Writing region close event to WAL at 1731522350376Closed at 1731522350376 2024-11-13T18:25:50,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741892_1070 (size=93) 2024-11-13T18:25:50,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741892_1070 (size=93) 2024-11-13T18:25:50,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741892_1070 (size=93) 2024-11-13T18:25:50,381 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:50,381 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43925281:(num 1731522350331) 2024-11-13T18:25:50,382 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:50,383 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:50,405 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, exclude list is [], retry=0 2024-11-13T18:25:50,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:50,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:50,408 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:50,410 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 2024-11-13T18:25:50,411 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:25:50,411 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 93136835b15ffd96067339b6dd8e3691, NAME => 'testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:50,411 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:50,411 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,411 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,413 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,414 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName a 2024-11-13T18:25:50,414 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,414 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,414 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,415 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName b 2024-11-13T18:25:50,415 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,416 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,416 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,417 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName c 2024-11-13T18:25:50,417 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,417 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,418 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,420 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,421 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,421 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,422 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:50,423 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,425 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93136835b15ffd96067339b6dd8e3691; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75024273, jitterRate=0.11794878542423248}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:50,426 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93136835b15ffd96067339b6dd8e3691: Writing region info on filesystem at 1731522350411Initializing all the Stores at 1731522350412 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350412Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350413 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350413Cleaning up temporary data from old regions at 1731522350421 (+8 ms)Region opened successfully at 1731522350426 (+5 ms) 2024-11-13T18:25:50,492 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 93136835b15ffd96067339b6dd8e3691 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T18:25:50,523 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/a/f9ff2e5a7f564984acc1a4fc13233953 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731522350426/Put/seqid=0 2024-11-13T18:25:50,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741895_1073 (size=5958) 2024-11-13T18:25:50,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741895_1073 (size=5958) 2024-11-13T18:25:50,554 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741895_1073 (size=5958) 2024-11-13T18:25:50,558 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/a/f9ff2e5a7f564984acc1a4fc13233953 2024-11-13T18:25:50,580 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/317f2e7cfdb44947b7d662d1d14956a6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731522350447/Put/seqid=0 2024-11-13T18:25:50,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741896_1074 (size=5958) 2024-11-13T18:25:50,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741896_1074 (size=5958) 2024-11-13T18:25:50,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741896_1074 (size=5958) 2024-11-13T18:25:50,589 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/317f2e7cfdb44947b7d662d1d14956a6 2024-11-13T18:25:50,618 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/c/5b00e78c19424f67951da1fa3c555023 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731522350465/Put/seqid=0 2024-11-13T18:25:50,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741897_1075 (size=5958) 2024-11-13T18:25:50,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741897_1075 (size=5958) 2024-11-13T18:25:50,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741897_1075 (size=5958) 2024-11-13T18:25:50,627 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/c/5b00e78c19424f67951da1fa3c555023 2024-11-13T18:25:50,634 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/a/f9ff2e5a7f564984acc1a4fc13233953 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/a/f9ff2e5a7f564984acc1a4fc13233953 2024-11-13T18:25:50,639 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/a/f9ff2e5a7f564984acc1a4fc13233953, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T18:25:50,640 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/317f2e7cfdb44947b7d662d1d14956a6 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/317f2e7cfdb44947b7d662d1d14956a6 2024-11-13T18:25:50,645 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/317f2e7cfdb44947b7d662d1d14956a6, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T18:25:50,646 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/c/5b00e78c19424f67951da1fa3c555023 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/c/5b00e78c19424f67951da1fa3c555023 2024-11-13T18:25:50,650 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/c/5b00e78c19424f67951da1fa3c555023, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T18:25:50,652 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 93136835b15ffd96067339b6dd8e3691 in 160ms, sequenceid=33, compaction requested=false 2024-11-13T18:25:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 93136835b15ffd96067339b6dd8e3691: 2024-11-13T18:25:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 93136835b15ffd96067339b6dd8e3691, disabling compactions & flushes 2024-11-13T18:25:50,652 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. after waiting 0 ms 2024-11-13T18:25:50,652 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,657 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. 2024-11-13T18:25:50,657 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 93136835b15ffd96067339b6dd8e3691: Waiting for close lock at 1731522350652Disabling compacts and flushes for region at 1731522350652Disabling writes for close at 1731522350652Writing region close event to WAL at 1731522350657 (+5 ms)Closed at 1731522350657 2024-11-13T18:25:50,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741894_1072 (size=5937) 2024-11-13T18:25:50,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741894_1072 (size=5937) 2024-11-13T18:25:50,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741894_1072 (size=5937) 2024-11-13T18:25:50,666 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/317f2e7cfdb44947b7d662d1d14956a6 to hdfs://localhost:40367/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/317f2e7cfdb44947b7d662d1d14956a6 2024-11-13T18:25:50,690 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, size=5.8 K (5937bytes) 2024-11-13T18:25:50,690 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 2024-11-13T18:25:50,691 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 after 1ms 2024-11-13T18:25:50,693 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 took 3ms 2024-11-13T18:25:50,700 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 so closing down 2024-11-13T18:25:50,701 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:25:50,701 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731522350384.temp 2024-11-13T18:25:50,703 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp 2024-11-13T18:25:50,703 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:25:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741898_1076 (size=5223) 2024-11-13T18:25:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741898_1076 (size=5223) 2024-11-13T18:25:50,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741898_1076 (size=5223) 2024-11-13T18:25:50,715 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T18:25:50,717 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 2024-11-13T18:25:50,717 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-13T18:25:50,717 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, journal: Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, size=5.8 K (5937bytes) at 1731522350690Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 so closing down at 1731522350700 (+10 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp at 1731522350703 (+3 ms)3 split writer threads finished at 1731522350703Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731522350715 (+12 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000003-wal.1731522350384.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 at 1731522350717 (+2 ms)Processed 32 edits across 1 Regions in 23 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731522350717 2024-11-13T18:25:50,719 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350384 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522350384 2024-11-13T18:25:50,719 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 2024-11-13T18:25:50,720 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:50,721 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:50,736 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350722, exclude list is [], retry=0 2024-11-13T18:25:50,740 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:50,740 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:50,741 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:50,745 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350722 2024-11-13T18:25:50,746 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 93136835b15ffd96067339b6dd8e3691, NAME => 'testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,746 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,748 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,749 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName a 2024-11-13T18:25:50,749 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,754 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/a/f9ff2e5a7f564984acc1a4fc13233953 2024-11-13T18:25:50,754 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,755 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,756 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName b 2024-11-13T18:25:50,756 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,756 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,757 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,757 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 93136835b15ffd96067339b6dd8e3691 columnFamilyName c 2024-11-13T18:25:50,757 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,766 DEBUG [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/c/5b00e78c19424f67951da1fa3c555023 2024-11-13T18:25:50,766 INFO [StoreOpener-93136835b15ffd96067339b6dd8e3691-1 {}] regionserver.HStore(327): Store=93136835b15ffd96067339b6dd8e3691/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,766 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,767 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,768 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,769 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 2024-11-13T18:25:50,772 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 2024-11-13T18:25:50,772 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 93136835b15ffd96067339b6dd8e3691 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T18:25:50,788 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/ebb733a77d00416b9d873a815eaa1b90 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731522350447/Put/seqid=0 2024-11-13T18:25:50,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741900_1078 (size=5958) 2024-11-13T18:25:50,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741900_1078 (size=5958) 2024-11-13T18:25:50,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741900_1078 (size=5958) 2024-11-13T18:25:50,796 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/ebb733a77d00416b9d873a815eaa1b90 2024-11-13T18:25:50,802 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/.tmp/b/ebb733a77d00416b9d873a815eaa1b90 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/ebb733a77d00416b9d873a815eaa1b90 2024-11-13T18:25:50,808 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/b/ebb733a77d00416b9d873a815eaa1b90, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T18:25:50,808 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 93136835b15ffd96067339b6dd8e3691 in 36ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T18:25:50,809 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/0000000000000000032 2024-11-13T18:25:50,810 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,810 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,811 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:50,812 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 93136835b15ffd96067339b6dd8e3691 2024-11-13T18:25:50,814 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/93136835b15ffd96067339b6dd8e3691/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T18:25:50,815 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 93136835b15ffd96067339b6dd8e3691; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63878088, jitterRate=-0.048142313957214355}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:50,816 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 93136835b15ffd96067339b6dd8e3691: Writing region info on filesystem at 1731522350746Initializing all the Stores at 1731522350747 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350747Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350748 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350748Obtaining lock to block concurrent updates at 1731522350773 (+25 ms)Preparing flush snapshotting stores in 93136835b15ffd96067339b6dd8e3691 at 1731522350773Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731522350773Flushing stores of testReplayEditsWrittenViaHRegion,,1731522350326.93136835b15ffd96067339b6dd8e3691. at 1731522350773Flushing 93136835b15ffd96067339b6dd8e3691/b: creating writer at 1731522350773Flushing 93136835b15ffd96067339b6dd8e3691/b: appending metadata at 1731522350787 (+14 ms)Flushing 93136835b15ffd96067339b6dd8e3691/b: closing flushed file at 1731522350787Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@12711870: reopening flushed file at 1731522350801 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 93136835b15ffd96067339b6dd8e3691 in 36ms, sequenceid=32, compaction requested=false; wal=null at 1731522350808 (+7 ms)Cleaning up temporary data from old regions at 1731522350810 (+2 ms)Region opened successfully at 1731522350816 (+6 ms) 2024-11-13T18:25:50,838 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=414 (was 406) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:47638 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:48648 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:48474 [Waiting for operation #46] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:58544 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:47518 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:58412 [Waiting for operation #33] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1127 (was 1051) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=536 (was 536), ProcessCount=11 (was 11), AvailableMemoryMB=2469 (was 2492) 2024-11-13T18:25:50,838 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T18:25:50,850 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=414, OpenFileDescriptor=1127, MaxFileDescriptor=1048576, SystemLoadAverage=536, ProcessCount=11, AvailableMemoryMB=2468 2024-11-13T18:25:50,850 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1127 is superior to 1024 2024-11-13T18:25:50,865 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:50,867 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:25:50,868 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:25:50,871 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-16694333, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-16694333, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:50,883 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-16694333/hregion-16694333.1731522350871, exclude list is [], retry=0 2024-11-13T18:25:50,886 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:50,887 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:50,887 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:50,891 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-16694333/hregion-16694333.1731522350871 2024-11-13T18:25:50,891 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:50,891 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 9466c04c1440fe54cb9f5bc7b44c8478, NAME => 'testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:25:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741902_1080 (size=68) 2024-11-13T18:25:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741902_1080 (size=68) 2024-11-13T18:25:50,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741902_1080 (size=68) 2024-11-13T18:25:50,903 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:50,905 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,906 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName a 2024-11-13T18:25:50,906 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,907 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,907 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,908 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName b 2024-11-13T18:25:50,908 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,908 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,908 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,910 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName c 2024-11-13T18:25:50,910 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:50,910 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:50,911 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,913 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,913 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,913 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:50,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:50,916 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:25:50,917 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9466c04c1440fe54cb9f5bc7b44c8478; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61810842, jitterRate=-0.07894667983055115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:50,917 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9466c04c1440fe54cb9f5bc7b44c8478: Writing region info on filesystem at 1731522350903Initializing all the Stores at 1731522350904 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350904Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350905 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522350905Cleaning up temporary data from old regions at 1731522350913 (+8 ms)Region opened successfully at 1731522350917 (+4 ms) 2024-11-13T18:25:50,918 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9466c04c1440fe54cb9f5bc7b44c8478, disabling compactions & flushes 2024-11-13T18:25:50,918 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:25:50,918 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:25:50,918 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. after waiting 0 ms 2024-11-13T18:25:50,918 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:25:50,918 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:25:50,918 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9466c04c1440fe54cb9f5bc7b44c8478: Waiting for close lock at 1731522350918Disabling compacts and flushes for region at 1731522350918Disabling writes for close at 1731522350918Writing region close event to WAL at 1731522350918Closed at 1731522350918 2024-11-13T18:25:50,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741901_1079 (size=93) 2024-11-13T18:25:50,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741901_1079 (size=93) 2024-11-13T18:25:50,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741901_1079 (size=93) 2024-11-13T18:25:50,924 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:25:50,925 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-16694333:(num 1731522350871) 2024-11-13T18:25:50,925 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:25:50,927 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:25:50,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, exclude list is [], retry=0 2024-11-13T18:25:50,944 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:25:50,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:25:50,945 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:25:50,947 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 2024-11-13T18:25:50,947 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:25:51,015 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9466c04c1440fe54cb9f5bc7b44c8478, NAME => 'testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:25:51,019 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,019 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:25:51,019 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,019 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,021 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,022 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName a 2024-11-13T18:25:51,023 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:51,023 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:51,023 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,024 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName b 2024-11-13T18:25:51,024 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:51,025 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:51,025 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,026 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName c 2024-11-13T18:25:51,026 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:25:51,026 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:25:51,026 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,027 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,028 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,030 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,030 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,031 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:25:51,032 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,033 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9466c04c1440fe54cb9f5bc7b44c8478; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67134113, jitterRate=3.7623941898345947E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:25:51,034 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9466c04c1440fe54cb9f5bc7b44c8478: Running coprocessor pre-open hook at 1731522351020Writing region info on filesystem at 1731522351020Initializing all the Stores at 1731522351021 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522351021Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522351021Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522351021Cleaning up temporary data from old regions at 1731522351030 (+9 ms)Running coprocessor post-open hooks at 1731522351034 (+4 ms)Region opened successfully at 1731522351034 2024-11-13T18:25:51,048 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9466c04c1440fe54cb9f5bc7b44c8478 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T18:25:51,049 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,050 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:52,273 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:25:53,051 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:54,052 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:55,052 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:56,053 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:56,464 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T18:25:57,053 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:57,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T18:25:57,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:57,880 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T18:25:57,880 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T18:25:58,054 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:25:59,055 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:00,056 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 9466c04c1440fe54cb9f5bc7b44c8478/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:00,057 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9466c04c1440fe54cb9f5bc7b44c8478: 2024-11-13T18:26:00,057 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,072 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 9466c04c1440fe54cb9f5bc7b44c8478: 2024-11-13T18:26:00,072 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T18:26:00,073 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 9466c04c1440fe54cb9f5bc7b44c8478, disabling compactions & flushes 2024-11-13T18:26:00,073 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,073 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,073 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. after waiting 0 ms 2024-11-13T18:26:00,073 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,077 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,077 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. 2024-11-13T18:26:00,077 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 9466c04c1440fe54cb9f5bc7b44c8478: Waiting for close lock at 1731522360072Running coprocessor pre-close hooks at 1731522360072Disabling compacts and flushes for region at 1731522360072Disabling writes for close at 1731522360073 (+1 ms)Writing region close event to WAL at 1731522360076 (+3 ms)Running coprocessor post-close hooks at 1731522360077 (+1 ms)Closed at 1731522360077 2024-11-13T18:26:00,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741903_1081 (size=3941) 2024-11-13T18:26:00,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741903_1081 (size=3941) 2024-11-13T18:26:00,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741903_1081 (size=3941) 2024-11-13T18:26:00,096 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, size=3.8 K (3941bytes) 2024-11-13T18:26:00,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 2024-11-13T18:26:00,097 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 after 0ms 2024-11-13T18:26:00,101 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 took 5ms 2024-11-13T18:26:00,103 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 so closing down 2024-11-13T18:26:00,103 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:26:00,105 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731522350928.temp 2024-11-13T18:26:00,106 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp 2024-11-13T18:26:00,106 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:26:00,119 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741904_1082 (size=2983) 2024-11-13T18:26:00,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741904_1082 (size=2983) 2024-11-13T18:26:00,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741904_1082 (size=2983) 2024-11-13T18:26:00,121 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T18:26:00,122 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 2024-11-13T18:26:00,122 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-13T18:26:00,122 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, journal: Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, size=3.8 K (3941bytes) at 1731522360097Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 so closing down at 1731522360103 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp at 1731522360106 (+3 ms)3 split writer threads finished at 1731522360106Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731522360121 (+15 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000004-wal.1731522350928.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 at 1731522360122 (+1 ms)Processed 23 edits across 1 Regions in 21 ms; skipped=3; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731522360122 2024-11-13T18:26:00,124 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522350928 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522350928 2024-11-13T18:26:00,125 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 2024-11-13T18:26:00,125 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:00,127 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:00,141 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522360127, exclude list is [], retry=0 2024-11-13T18:26:00,143 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:00,144 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:00,144 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:00,147 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522360127 2024-11-13T18:26:00,151 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:26:00,152 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 9466c04c1440fe54cb9f5bc7b44c8478, NAME => 'testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:00,152 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,152 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:00,153 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,153 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,155 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,156 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName a 2024-11-13T18:26:00,156 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:00,156 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:00,157 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,158 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName b 2024-11-13T18:26:00,158 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:00,158 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:00,158 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,159 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 9466c04c1440fe54cb9f5bc7b44c8478 columnFamilyName c 2024-11-13T18:26:00,159 DEBUG [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:00,160 INFO [StoreOpener-9466c04c1440fe54cb9f5bc7b44c8478-1 {}] regionserver.HStore(327): Store=9466c04c1440fe54cb9f5bc7b44c8478/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:00,160 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,161 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,162 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,163 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 2024-11-13T18:26:00,167 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 2024-11-13T18:26:00,167 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 9466c04c1440fe54cb9f5bc7b44c8478 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T18:26:00,186 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/a/8f6880ea40034b03849d6997f621f9a8 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731522360063/Put/seqid=0 2024-11-13T18:26:00,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741906_1084 (size=5523) 2024-11-13T18:26:00,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741906_1084 (size=5523) 2024-11-13T18:26:00,204 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/a/8f6880ea40034b03849d6997f621f9a8 2024-11-13T18:26:00,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741906_1084 (size=5523) 2024-11-13T18:26:00,230 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/b/affc6ca8cc9b4b32be50345ad8ab4683 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731522360057/Put/seqid=0 2024-11-13T18:26:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741907_1085 (size=5524) 2024-11-13T18:26:00,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741907_1085 (size=5524) 2024-11-13T18:26:00,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741907_1085 (size=5524) 2024-11-13T18:26:00,243 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/b/affc6ca8cc9b4b32be50345ad8ab4683 2024-11-13T18:26:00,272 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/c/5a5ba9bb36954608a2d17aca8f6663a5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731522360060/Put/seqid=0 2024-11-13T18:26:00,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741908_1086 (size=5457) 2024-11-13T18:26:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741908_1086 (size=5457) 2024-11-13T18:26:00,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741908_1086 (size=5457) 2024-11-13T18:26:00,288 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/c/5a5ba9bb36954608a2d17aca8f6663a5 2024-11-13T18:26:00,295 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/a/8f6880ea40034b03849d6997f621f9a8 as hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/a/8f6880ea40034b03849d6997f621f9a8 2024-11-13T18:26:00,302 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/a/8f6880ea40034b03849d6997f621f9a8, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T18:26:00,303 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/b/affc6ca8cc9b4b32be50345ad8ab4683 as hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/b/affc6ca8cc9b4b32be50345ad8ab4683 2024-11-13T18:26:00,310 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/b/affc6ca8cc9b4b32be50345ad8ab4683, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T18:26:00,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/.tmp/c/5a5ba9bb36954608a2d17aca8f6663a5 as hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/c/5a5ba9bb36954608a2d17aca8f6663a5 2024-11-13T18:26:00,319 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/c/5a5ba9bb36954608a2d17aca8f6663a5, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T18:26:00,319 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 9466c04c1440fe54cb9f5bc7b44c8478 in 152ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T18:26:00,320 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/0000000000000000026 2024-11-13T18:26:00,321 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,321 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,322 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:00,323 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,326 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsAfterAbortingFlush/9466c04c1440fe54cb9f5bc7b44c8478/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T18:26:00,326 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 9466c04c1440fe54cb9f5bc7b44c8478; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59730701, jitterRate=-0.1099431961774826}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:00,327 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 9466c04c1440fe54cb9f5bc7b44c8478 2024-11-13T18:26:00,327 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 9466c04c1440fe54cb9f5bc7b44c8478: Running coprocessor pre-open hook at 1731522360153Writing region info on filesystem at 1731522360153Initializing all the Stores at 1731522360154 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522360154Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522360154Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522360154Obtaining lock to block concurrent updates at 1731522360168 (+14 ms)Preparing flush snapshotting stores in 9466c04c1440fe54cb9f5bc7b44c8478 at 1731522360168Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731522360168Flushing stores of testReplayEditsAfterAbortingFlush,,1731522350866.9466c04c1440fe54cb9f5bc7b44c8478. at 1731522360168Flushing 9466c04c1440fe54cb9f5bc7b44c8478/a: creating writer at 1731522360168Flushing 9466c04c1440fe54cb9f5bc7b44c8478/a: appending metadata at 1731522360185 (+17 ms)Flushing 9466c04c1440fe54cb9f5bc7b44c8478/a: closing flushed file at 1731522360185Flushing 9466c04c1440fe54cb9f5bc7b44c8478/b: creating writer at 1731522360211 (+26 ms)Flushing 9466c04c1440fe54cb9f5bc7b44c8478/b: appending metadata at 1731522360229 (+18 ms)Flushing 9466c04c1440fe54cb9f5bc7b44c8478/b: closing flushed file at 1731522360229Flushing 9466c04c1440fe54cb9f5bc7b44c8478/c: creating writer at 1731522360250 (+21 ms)Flushing 9466c04c1440fe54cb9f5bc7b44c8478/c: appending metadata at 1731522360271 (+21 ms)Flushing 9466c04c1440fe54cb9f5bc7b44c8478/c: closing flushed file at 1731522360271Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71e51f07: reopening flushed file at 1731522360294 (+23 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@34fab709: reopening flushed file at 1731522360302 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21b0422: reopening flushed file at 1731522360310 (+8 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 9466c04c1440fe54cb9f5bc7b44c8478 in 152ms, sequenceid=26, compaction requested=false; wal=null at 1731522360319 (+9 ms)Cleaning up temporary data from old regions at 1731522360321 (+2 ms)Running coprocessor post-open hooks at 1731522360327 (+6 ms)Region opened successfully at 1731522360327 2024-11-13T18:26:00,355 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=420 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:46242 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:46216 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:57958 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:34632 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:34648 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: java.util.concurrent.ThreadPoolExecutor$Worker@4fba88d1[State = -1, empty queue] java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:57916 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1186 (was 1127) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 536), ProcessCount=11 (was 11), AvailableMemoryMB=2256 (was 2468) 2024-11-13T18:26:00,356 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-13T18:26:00,372 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=420, OpenFileDescriptor=1186, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=2256 2024-11-13T18:26:00,372 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1186 is superior to 1024 2024-11-13T18:26:00,390 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:00,393 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:00,393 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:26:00,396 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50157935, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-50157935, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:00,414 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50157935/hregion-50157935.1731522360396, exclude list is [], retry=0 2024-11-13T18:26:00,419 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:00,419 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:00,421 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:00,429 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50157935/hregion-50157935.1731522360396 2024-11-13T18:26:00,430 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:26:00,430 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 6be7352e9fa8d72e2708b8dcd24271ed, NAME => 'testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:26:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741910_1088 (size=61) 2024-11-13T18:26:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741910_1088 (size=61) 2024-11-13T18:26:00,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741910_1088 (size=61) 2024-11-13T18:26:00,457 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:00,459 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,461 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be7352e9fa8d72e2708b8dcd24271ed columnFamilyName a 2024-11-13T18:26:00,462 DEBUG [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:00,462 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(327): Store=6be7352e9fa8d72e2708b8dcd24271ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:00,462 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,463 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,464 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,464 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,466 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,468 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:26:00,469 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6be7352e9fa8d72e2708b8dcd24271ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62977325, jitterRate=-0.06156472861766815}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:26:00,469 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6be7352e9fa8d72e2708b8dcd24271ed: Writing region info on filesystem at 1731522360457Initializing all the Stores at 1731522360458 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522360459 (+1 ms)Cleaning up temporary data from old regions at 1731522360464 (+5 ms)Region opened successfully at 1731522360469 (+5 ms) 2024-11-13T18:26:00,469 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6be7352e9fa8d72e2708b8dcd24271ed, disabling compactions & flushes 2024-11-13T18:26:00,470 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,470 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,470 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. after waiting 0 ms 2024-11-13T18:26:00,470 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,470 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,470 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6be7352e9fa8d72e2708b8dcd24271ed: Waiting for close lock at 1731522360469Disabling compacts and flushes for region at 1731522360469Disabling writes for close at 1731522360470 (+1 ms)Writing region close event to WAL at 1731522360470Closed at 1731522360470 2024-11-13T18:26:00,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741909_1087 (size=93) 2024-11-13T18:26:00,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741909_1087 (size=93) 2024-11-13T18:26:00,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741909_1087 (size=93) 2024-11-13T18:26:00,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:26:00,477 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50157935:(num 1731522360396) 2024-11-13T18:26:00,477 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:00,479 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:00,497 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, exclude list is [], retry=0 2024-11-13T18:26:00,500 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:00,500 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:00,501 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:00,502 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 2024-11-13T18:26:00,503 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:26:00,503 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6be7352e9fa8d72e2708b8dcd24271ed, NAME => 'testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:00,503 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:00,503 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,503 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,509 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,510 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be7352e9fa8d72e2708b8dcd24271ed columnFamilyName a 2024-11-13T18:26:00,510 DEBUG [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:00,510 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(327): Store=6be7352e9fa8d72e2708b8dcd24271ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:00,510 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,511 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,513 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,513 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,513 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,515 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:00,516 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6be7352e9fa8d72e2708b8dcd24271ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62320554, jitterRate=-0.07135137915611267}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:26:00,517 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6be7352e9fa8d72e2708b8dcd24271ed: Writing region info on filesystem at 1731522360503Initializing all the Stores at 1731522360508 (+5 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522360508Cleaning up temporary data from old regions at 1731522360513 (+5 ms)Region opened successfully at 1731522360517 (+4 ms) 2024-11-13T18:26:00,531 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 6be7352e9fa8d72e2708b8dcd24271ed, disabling compactions & flushes 2024-11-13T18:26:00,531 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,531 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,531 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. after waiting 0 ms 2024-11-13T18:26:00,531 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,533 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,533 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. 2024-11-13T18:26:00,533 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 6be7352e9fa8d72e2708b8dcd24271ed: Waiting for close lock at 1731522360531Disabling compacts and flushes for region at 1731522360531Disabling writes for close at 1731522360531Writing region close event to WAL at 1731522360533 (+2 ms)Closed at 1731522360533 2024-11-13T18:26:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741911_1089 (size=1623) 2024-11-13T18:26:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741911_1089 (size=1623) 2024-11-13T18:26:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741911_1089 (size=1623) 2024-11-13T18:26:00,560 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, size=1.6 K (1623bytes) 2024-11-13T18:26:00,560 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 2024-11-13T18:26:00,561 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 after 0ms 2024-11-13T18:26:00,563 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 took 3ms 2024-11-13T18:26:00,565 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 so closing down 2024-11-13T18:26:00,565 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:26:00,567 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731522360480.temp 2024-11-13T18:26:00,568 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp 2024-11-13T18:26:00,568 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:26:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741912_1090 (size=1623) 2024-11-13T18:26:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741912_1090 (size=1623) 2024-11-13T18:26:00,578 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741912_1090 (size=1623) 2024-11-13T18:26:00,579 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T18:26:00,580 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp to hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 2024-11-13T18:26:00,580 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-13T18:26:00,580 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, journal: Splitting hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, size=1.6 K (1623bytes) at 1731522360560Finishing writing output for hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 so closing down at 1731522360565 (+5 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp at 1731522360568 (+3 ms)3 split writer threads finished at 1731522360568Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731522360579 (+11 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000003-wal.1731522360480.temp to hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 at 1731522360580 (+1 ms)Processed 10 edits across 1 Regions in 16 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731522360580 2024-11-13T18:26:00,582 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522360480 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522360480 2024-11-13T18:26:00,583 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 2024-11-13T18:26:01,022 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:01,025 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:01,043 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522361025, exclude list is [], retry=0 2024-11-13T18:26:01,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:01,056 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:01,057 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:01,093 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522361025 2024-11-13T18:26:01,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:26:01,093 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6be7352e9fa8d72e2708b8dcd24271ed, NAME => 'testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:01,094 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:01,094 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,094 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,107 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,110 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be7352e9fa8d72e2708b8dcd24271ed columnFamilyName a 2024-11-13T18:26:01,110 DEBUG [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,112 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(327): Store=6be7352e9fa8d72e2708b8dcd24271ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,113 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,114 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,115 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,117 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 2024-11-13T18:26:01,122 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 2024-11-13T18:26:01,122 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 6be7352e9fa8d72e2708b8dcd24271ed 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T18:26:01,147 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/.tmp/a/79e06fe7da81471d88e53da5aaa34989 is 79, key is testDatalossWhenInputError/a:x0/1731522360517/Put/seqid=0 2024-11-13T18:26:01,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741914_1092 (size=5808) 2024-11-13T18:26:01,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741914_1092 (size=5808) 2024-11-13T18:26:01,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741914_1092 (size=5808) 2024-11-13T18:26:01,169 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/.tmp/a/79e06fe7da81471d88e53da5aaa34989 2024-11-13T18:26:01,182 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/.tmp/a/79e06fe7da81471d88e53da5aaa34989 as hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/a/79e06fe7da81471d88e53da5aaa34989 2024-11-13T18:26:01,193 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/a/79e06fe7da81471d88e53da5aaa34989, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T18:26:01,193 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6be7352e9fa8d72e2708b8dcd24271ed in 71ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T18:26:01,194 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/0000000000000000012 2024-11-13T18:26:01,194 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,194 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,197 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,201 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T18:26:01,202 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6be7352e9fa8d72e2708b8dcd24271ed; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69677232, jitterRate=0.03827166557312012}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:26:01,203 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6be7352e9fa8d72e2708b8dcd24271ed: Writing region info on filesystem at 1731522361094Initializing all the Stores at 1731522361106 (+12 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361107 (+1 ms)Obtaining lock to block concurrent updates at 1731522361122 (+15 ms)Preparing flush snapshotting stores in 6be7352e9fa8d72e2708b8dcd24271ed at 1731522361122Finished memstore snapshotting testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731522361123 (+1 ms)Flushing stores of testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed. at 1731522361123Flushing 6be7352e9fa8d72e2708b8dcd24271ed/a: creating writer at 1731522361123Flushing 6be7352e9fa8d72e2708b8dcd24271ed/a: appending metadata at 1731522361146 (+23 ms)Flushing 6be7352e9fa8d72e2708b8dcd24271ed/a: closing flushed file at 1731522361146Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@692dbcde: reopening flushed file at 1731522361178 (+32 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 6be7352e9fa8d72e2708b8dcd24271ed in 71ms, sequenceid=12, compaction requested=false; wal=null at 1731522361193 (+15 ms)Cleaning up temporary data from old regions at 1731522361194 (+1 ms)Region opened successfully at 1731522361203 (+9 ms) 2024-11-13T18:26:01,206 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 6be7352e9fa8d72e2708b8dcd24271ed, NAME => 'testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:01,207 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731522360391.6be7352e9fa8d72e2708b8dcd24271ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:01,207 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,207 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,213 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,214 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6be7352e9fa8d72e2708b8dcd24271ed columnFamilyName a 2024-11-13T18:26:01,214 DEBUG [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,222 DEBUG [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/a/79e06fe7da81471d88e53da5aaa34989 2024-11-13T18:26:01,222 INFO [StoreOpener-6be7352e9fa8d72e2708b8dcd24271ed-1 {}] regionserver.HStore(327): Store=6be7352e9fa8d72e2708b8dcd24271ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,222 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,224 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,225 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,225 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,225 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,230 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 6be7352e9fa8d72e2708b8dcd24271ed 2024-11-13T18:26:01,235 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testDatalossWhenInputError/6be7352e9fa8d72e2708b8dcd24271ed/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T18:26:01,237 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 6be7352e9fa8d72e2708b8dcd24271ed; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61565496, jitterRate=-0.0826026201248169}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T18:26:01,237 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 6be7352e9fa8d72e2708b8dcd24271ed: Writing region info on filesystem at 1731522361207Initializing all the Stores at 1731522361208 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361208Cleaning up temporary data from old regions at 1731522361225 (+17 ms)Region opened successfully at 1731522361237 (+12 ms) 2024-11-13T18:26:01,264 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=430 (was 420) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:57998 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:46310 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:46216 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:57958 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:34648 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:34728 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1268 (was 1186) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=476 (was 476), ProcessCount=11 (was 11), AvailableMemoryMB=2217 (was 2256) 2024-11-13T18:26:01,264 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-13T18:26:01,281 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=430, OpenFileDescriptor=1268, MaxFileDescriptor=1048576, SystemLoadAverage=476, ProcessCount=11, AvailableMemoryMB=2215 2024-11-13T18:26:01,281 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1268 is superior to 1024 2024-11-13T18:26:01,311 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:01,314 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:01,315 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:26:01,319 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79455980, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-79455980, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:01,340 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79455980/hregion-79455980.1731522361320, exclude list is [], retry=0 2024-11-13T18:26:01,349 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:01,349 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:01,353 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:01,367 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79455980/hregion-79455980.1731522361320 2024-11-13T18:26:01,368 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:26:01,369 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 476af033269e9c26ec0d31e01a7c3812, NAME => 'testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:26:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741916_1094 (size=63) 2024-11-13T18:26:01,390 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741916_1094 (size=63) 2024-11-13T18:26:01,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741916_1094 (size=63) 2024-11-13T18:26:01,794 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:01,797 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,799 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName a 2024-11-13T18:26:01,799 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,800 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,800 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,802 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName b 2024-11-13T18:26:01,802 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,803 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,803 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,804 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName c 2024-11-13T18:26:01,804 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,805 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,805 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,806 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,808 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,808 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,808 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:01,810 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,812 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:26:01,813 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 476af033269e9c26ec0d31e01a7c3812; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62707248, jitterRate=-0.06558918952941895}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 476af033269e9c26ec0d31e01a7c3812: Writing region info on filesystem at 1731522361794Initializing all the Stores at 1731522361797 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361797Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361797Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361797Cleaning up temporary data from old regions at 1731522361808 (+11 ms)Region opened successfully at 1731522361814 (+6 ms) 2024-11-13T18:26:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 476af033269e9c26ec0d31e01a7c3812, disabling compactions & flushes 2024-11-13T18:26:01,814 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. after waiting 0 ms 2024-11-13T18:26:01,814 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:01,815 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:01,815 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 476af033269e9c26ec0d31e01a7c3812: Waiting for close lock at 1731522361814Disabling compacts and flushes for region at 1731522361814Disabling writes for close at 1731522361814Writing region close event to WAL at 1731522361815 (+1 ms)Closed at 1731522361815 2024-11-13T18:26:01,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741915_1093 (size=93) 2024-11-13T18:26:01,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741915_1093 (size=93) 2024-11-13T18:26:01,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741915_1093 (size=93) 2024-11-13T18:26:01,838 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:26:01,838 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79455980:(num 1731522361320) 2024-11-13T18:26:01,838 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:01,840 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:01,857 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, exclude list is [], retry=0 2024-11-13T18:26:01,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:01,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:01,862 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:01,883 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 2024-11-13T18:26:01,883 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:26:01,884 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 476af033269e9c26ec0d31e01a7c3812, NAME => 'testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:01,884 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:01,884 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,884 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,886 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,887 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName a 2024-11-13T18:26:01,888 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,888 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,889 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,892 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName b 2024-11-13T18:26:01,892 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,895 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,895 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,897 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName c 2024-11-13T18:26:01,897 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:01,898 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:01,898 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,899 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,901 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,903 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,903 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,907 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:01,914 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:01,915 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 476af033269e9c26ec0d31e01a7c3812; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63361285, jitterRate=-0.055843278765678406}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:01,916 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 476af033269e9c26ec0d31e01a7c3812: Writing region info on filesystem at 1731522361884Initializing all the Stores at 1731522361886 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361886Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361886Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522361886Cleaning up temporary data from old regions at 1731522361903 (+17 ms)Region opened successfully at 1731522361916 (+13 ms) 2024-11-13T18:26:01,934 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731522361933/Put/seqid=0 2024-11-13T18:26:01,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741918_1096 (size=4875) 2024-11-13T18:26:01,998 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741918_1096 (size=4875) 2024-11-13T18:26:02,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741918_1096 (size=4875) 2024-11-13T18:26:02,006 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731522362005/Put/seqid=0 2024-11-13T18:26:02,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741919_1097 (size=4875) 2024-11-13T18:26:02,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741919_1097 (size=4875) 2024-11-13T18:26:02,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741919_1097 (size=4875) 2024-11-13T18:26:02,063 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731522362063/Put/seqid=0 2024-11-13T18:26:02,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741920_1098 (size=4875) 2024-11-13T18:26:02,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741920_1098 (size=4875) 2024-11-13T18:26:02,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741920_1098 (size=4875) 2024-11-13T18:26:02,105 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,118 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T18:26:02,118 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T18:26:02,118 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,126 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T18:26:02,126 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T18:26:02,127 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,135 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T18:26:02,135 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T18:26:02,136 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 476af033269e9c26ec0d31e01a7c3812 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T18:26:02,177 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp/a/7770114e40844364aff5d688e275f5c1 is 55, key is testCompactedBulkLoadedFiles/a:a/1731522361916/Put/seqid=0 2024-11-13T18:26:02,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741921_1099 (size=5107) 2024-11-13T18:26:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741921_1099 (size=5107) 2024-11-13T18:26:02,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741921_1099 (size=5107) 2024-11-13T18:26:02,234 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp/a/7770114e40844364aff5d688e275f5c1 2024-11-13T18:26:02,244 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp/a/7770114e40844364aff5d688e275f5c1 as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1 2024-11-13T18:26:02,253 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T18:26:02,256 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 476af033269e9c26ec0d31e01a7c3812 in 121ms, sequenceid=4, compaction requested=false 2024-11-13T18:26:02,256 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 476af033269e9c26ec0d31e01a7c3812: 2024-11-13T18:26:02,259 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ 2024-11-13T18:26:02,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ 2024-11-13T18:26:02,264 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ 2024-11-13T18:26:02,265 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile0 into 476af033269e9c26ec0d31e01a7c3812/a as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ - updating store file list. 2024-11-13T18:26:02,271 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 574d5a7ecd34421098967b946b6ca3e3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:02,271 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ into 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,272 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile0 into 476af033269e9c26ec0d31e01a7c3812/a (new location: hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_) 2024-11-13T18:26:02,273 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile1 into 476af033269e9c26ec0d31e01a7c3812/a as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ - updating store file list. 2024-11-13T18:26:02,279 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5e74317229f44c43b4dc8c70f4672f61_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:02,279 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ into 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,279 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile1 into 476af033269e9c26ec0d31e01a7c3812/a (new location: hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_) 2024-11-13T18:26:02,289 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile2 into 476af033269e9c26ec0d31e01a7c3812/a as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ - updating store file list. 2024-11-13T18:26:02,303 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e6ade00201f24a759a9e105ffee49531_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:02,303 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ into 476af033269e9c26ec0d31e01a7c3812/a 2024-11-13T18:26:02,303 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:40367/hbase/testCompactedBulkLoadedFiles/hfile2 into 476af033269e9c26ec0d31e01a7c3812/a (new location: hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_) 2024-11-13T18:26:02,323 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T18:26:02,323 DEBUG [Time-limited test {}] regionserver.HStore(1541): 476af033269e9c26ec0d31e01a7c3812/a is initiating major compaction (all files) 2024-11-13T18:26:02,323 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 476af033269e9c26ec0d31e01a7c3812/a in testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:02,324 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_] into tmpdir=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp, totalSize=19.3 K 2024-11-13T18:26:02,324 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 7770114e40844364aff5d688e275f5c1, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731522361916 2024-11-13T18:26:02,325 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 574d5a7ecd34421098967b946b6ca3e3_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T18:26:02,325 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5e74317229f44c43b4dc8c70f4672f61_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T18:26:02,329 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e6ade00201f24a759a9e105ffee49531_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T18:26:02,367 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp/a/cc69aa15f13640a997c513072ccb44b5 is 55, key is testCompactedBulkLoadedFiles/a:a/1731522361916/Put/seqid=0 2024-11-13T18:26:02,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741922_1100 (size=6154) 2024-11-13T18:26:02,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741922_1100 (size=6154) 2024-11-13T18:26:02,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741922_1100 (size=6154) 2024-11-13T18:26:02,396 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/.tmp/a/cc69aa15f13640a997c513072ccb44b5 as hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/cc69aa15f13640a997c513072ccb44b5 2024-11-13T18:26:02,403 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 476af033269e9c26ec0d31e01a7c3812/a of 476af033269e9c26ec0d31e01a7c3812 into cc69aa15f13640a997c513072ccb44b5(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T18:26:02,403 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 476af033269e9c26ec0d31e01a7c3812: 2024-11-13T18:26:02,403 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T18:26:02,403 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T18:26:02,439 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, size=0 (0bytes) 2024-11-13T18:26:02,439 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 might be still open, length is 0 2024-11-13T18:26:02,439 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 2024-11-13T18:26:02,440 WARN [IPC Server handler 3 on default port 40367 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-11-13T18:26:02,440 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 after 1ms 2024-11-13T18:26:05,228 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:46344 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:41041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46344 dst: /127.0.0.1:41041 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41041 remote=/127.0.0.1:46344]. Total timeout mills is 60000, 57174 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:58046 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:46817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58046 dst: /127.0.0.1:46817 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:34766 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:33523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34766 dst: /127.0.0.1:33523 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:05,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741917_1101 (size=1659) 2024-11-13T18:26:05,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741917_1101 (size=1659) 2024-11-13T18:26:05,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741917_1101 (size=1659) 2024-11-13T18:26:05,607 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T18:26:06,441 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 after 4002ms 2024-11-13T18:26:06,444 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 took 4005ms 2024-11-13T18:26:06,447 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841; continuing. 2024-11-13T18:26:06,447 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 so closing down 2024-11-13T18:26:06,447 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:26:06,449 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731522361841.temp 2024-11-13T18:26:06,450 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp 2024-11-13T18:26:06,450 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:26:06,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741923_1102 (size=698) 2024-11-13T18:26:06,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741923_1102 (size=698) 2024-11-13T18:26:06,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741923_1102 (size=698) 2024-11-13T18:26:06,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T18:26:06,460 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp to hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 2024-11-13T18:26:06,460 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T18:26:06,460 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, journal: Splitting hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, size=0 (0bytes) at 1731522362439Finishing writing output for hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 so closing down at 1731522366447 (+4008 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp at 1731522366450 (+3 ms)3 split writer threads finished at 1731522366450Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731522366458 (+8 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000003-wal.1731522361841.temp to hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 at 1731522366460 (+2 ms)Processed 5 edits across 1 Regions in 15 ms; skipped=3; WAL=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841, size=0, length=0, corrupted=false, cancelled=false at 1731522366460 2024-11-13T18:26:06,461 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522361841 2024-11-13T18:26:06,462 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 2024-11-13T18:26:06,462 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:06,464 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:06,485 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522366465, exclude list is [], retry=0 2024-11-13T18:26:06,488 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:06,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:06,489 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:06,491 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522366465 2024-11-13T18:26:06,491 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:26:06,491 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 476af033269e9c26ec0d31e01a7c3812, NAME => 'testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,492 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,493 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,494 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName a 2024-11-13T18:26:06,494 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,505 DEBUG [StoreFileOpener-476af033269e9c26ec0d31e01a7c3812-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 574d5a7ecd34421098967b946b6ca3e3_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:06,505 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ 2024-11-13T18:26:06,509 DEBUG [StoreFileOpener-476af033269e9c26ec0d31e01a7c3812-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5e74317229f44c43b4dc8c70f4672f61_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:06,509 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ 2024-11-13T18:26:06,514 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1 2024-11-13T18:26:06,518 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/cc69aa15f13640a997c513072ccb44b5 2024-11-13T18:26:06,521 DEBUG [StoreFileOpener-476af033269e9c26ec0d31e01a7c3812-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e6ade00201f24a759a9e105ffee49531_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T18:26:06,521 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ 2024-11-13T18:26:06,521 WARN [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@44dd2123 2024-11-13T18:26:06,522 WARN [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@44dd2123 2024-11-13T18:26:06,522 WARN [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@44dd2123 2024-11-13T18:26:06,522 WARN [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@44dd2123 2024-11-13T18:26:06,522 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_] to archive 2024-11-13T18:26:06,523 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T18:26:06,525 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ to hdfs://localhost:40367/hbase/archive/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/574d5a7ecd34421098967b946b6ca3e3_SeqId_4_ 2024-11-13T18:26:06,527 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ to hdfs://localhost:40367/hbase/archive/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/5e74317229f44c43b4dc8c70f4672f61_SeqId_4_ 2024-11-13T18:26:06,528 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1 to hdfs://localhost:40367/hbase/archive/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/7770114e40844364aff5d688e275f5c1 2024-11-13T18:26:06,529 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ to hdfs://localhost:40367/hbase/archive/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/a/e6ade00201f24a759a9e105ffee49531_SeqId_4_ 2024-11-13T18:26:06,529 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,529 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,530 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName b 2024-11-13T18:26:06,530 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,531 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,532 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,533 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 476af033269e9c26ec0d31e01a7c3812 columnFamilyName c 2024-11-13T18:26:06,533 DEBUG [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,533 INFO [StoreOpener-476af033269e9c26ec0d31e01a7c3812-1 {}] regionserver.HStore(327): Store=476af033269e9c26ec0d31e01a7c3812/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,534 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,534 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,536 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,538 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 2024-11-13T18:26:06,547 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 476af033269e9c26ec0d31e01a7c3812 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "476af033269e9c26ec0d31e01a7c3812" family_name: "a" compaction_input: "7770114e40844364aff5d688e275f5c1" compaction_input: "574d5a7ecd34421098967b946b6ca3e3_SeqId_4_" compaction_input: "5e74317229f44c43b4dc8c70f4672f61_SeqId_4_" compaction_input: "e6ade00201f24a759a9e105ffee49531_SeqId_4_" compaction_output: "cc69aa15f13640a997c513072ccb44b5" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T18:26:06,547 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T18:26:06,547 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 2024-11-13T18:26:06,548 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/0000000000000000008 2024-11-13T18:26:06,557 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,557 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,559 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:06,561 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 476af033269e9c26ec0d31e01a7c3812 2024-11-13T18:26:06,565 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testCompactedBulkLoadedFiles/476af033269e9c26ec0d31e01a7c3812/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T18:26:06,566 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 476af033269e9c26ec0d31e01a7c3812; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63565948, jitterRate=-0.05279356241226196}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:06,567 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 476af033269e9c26ec0d31e01a7c3812: Writing region info on filesystem at 1731522366492Initializing all the Stores at 1731522366493 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366493Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366493Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366493Cleaning up temporary data from old regions at 1731522366557 (+64 ms)Region opened successfully at 1731522366567 (+10 ms) 2024-11-13T18:26:06,569 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 476af033269e9c26ec0d31e01a7c3812, disabling compactions & flushes 2024-11-13T18:26:06,569 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:06,569 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:06,569 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. after waiting 0 ms 2024-11-13T18:26:06,569 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:06,570 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731522361312.476af033269e9c26ec0d31e01a7c3812. 2024-11-13T18:26:06,570 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 476af033269e9c26ec0d31e01a7c3812: Waiting for close lock at 1731522366569Disabling compacts and flushes for region at 1731522366569Disabling writes for close at 1731522366569Writing region close event to WAL at 1731522366570 (+1 ms)Closed at 1731522366570 2024-11-13T18:26:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741924_1103 (size=93) 2024-11-13T18:26:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741924_1103 (size=93) 2024-11-13T18:26:06,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741924_1103 (size=93) 2024-11-13T18:26:06,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:26:06,575 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731522366465) 2024-11-13T18:26:06,591 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 430) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-322238331_22 at /127.0.0.1:38690 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-322238331_22 at /127.0.0.1:33308 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-322238331_22 at /127.0.0.1:59066 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:40367 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40367 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1268) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=510 (was 476) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2070 (was 2215) 2024-11-13T18:26:06,592 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T18:26:06,607 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=510, ProcessCount=11, AvailableMemoryMB=2068 2024-11-13T18:26:06,607 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T18:26:06,628 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:06,631 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T18:26:06,631 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T18:26:06,634 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-70691738, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/hregion-70691738, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:06,648 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-70691738/hregion-70691738.1731522366634, exclude list is [], retry=0 2024-11-13T18:26:06,650 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:06,651 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:06,651 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:06,653 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-70691738/hregion-70691738.1731522366634 2024-11-13T18:26:06,653 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085)] 2024-11-13T18:26:06,653 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 64ad6e58a5d351a0e777ed7ed62ea43e, NAME => 'testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40367/hbase 2024-11-13T18:26:06,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741926_1105 (size=67) 2024-11-13T18:26:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741926_1105 (size=67) 2024-11-13T18:26:06,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741926_1105 (size=67) 2024-11-13T18:26:06,662 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:06,664 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,666 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName a 2024-11-13T18:26:06,666 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,666 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,666 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,667 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName b 2024-11-13T18:26:06,667 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,668 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,668 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,669 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName c 2024-11-13T18:26:06,669 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,669 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,669 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,670 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,671 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,671 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,672 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:06,673 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,675 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T18:26:06,675 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64ad6e58a5d351a0e777ed7ed62ea43e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70400887, jitterRate=0.04905496537685394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:06,675 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Writing region info on filesystem at 1731522366663Initializing all the Stores at 1731522366663Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366663Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366664 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366664Cleaning up temporary data from old regions at 1731522366671 (+7 ms)Region opened successfully at 1731522366675 (+4 ms) 2024-11-13T18:26:06,675 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64ad6e58a5d351a0e777ed7ed62ea43e, disabling compactions & flushes 2024-11-13T18:26:06,676 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,676 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,676 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. after waiting 0 ms 2024-11-13T18:26:06,676 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,676 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,676 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Waiting for close lock at 1731522366675Disabling compacts and flushes for region at 1731522366675Disabling writes for close at 1731522366676 (+1 ms)Writing region close event to WAL at 1731522366676Closed at 1731522366676 2024-11-13T18:26:06,679 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-70691738/hregion-70691738.1731522366634 not finished, retry = 0 2024-11-13T18:26:06,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741925_1104 (size=93) 2024-11-13T18:26:06,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741925_1104 (size=93) 2024-11-13T18:26:06,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741925_1104 (size=93) 2024-11-13T18:26:06,781 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:26:06,782 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-70691738:(num 1731522366634) 2024-11-13T18:26:06,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:06,784 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:06,799 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, exclude list is [], retry=0 2024-11-13T18:26:06,801 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:06,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:06,802 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:06,804 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 2024-11-13T18:26:06,805 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:26:06,805 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 64ad6e58a5d351a0e777ed7ed62ea43e, NAME => 'testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:06,805 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:06,805 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,805 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,806 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,807 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName a 2024-11-13T18:26:06,807 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,808 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,808 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,809 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName b 2024-11-13T18:26:06,809 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,809 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,809 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,810 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName c 2024-11-13T18:26:06,810 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:06,810 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:06,810 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,812 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,813 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,814 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:06,815 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:06,816 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64ad6e58a5d351a0e777ed7ed62ea43e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66823034, jitterRate=-0.004259198904037476}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:06,816 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Writing region info on filesystem at 1731522366805Initializing all the Stores at 1731522366806 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366806Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366806Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522366806Cleaning up temporary data from old regions at 1731522366813 (+7 ms)Region opened successfully at 1731522366816 (+3 ms) 2024-11-13T18:26:06,823 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64ad6e58a5d351a0e777ed7ed62ea43e 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T18:26:06,841 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/9520bc72859f46939319585067198449 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731522366816/Put/seqid=0 2024-11-13T18:26:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741928_1107 (size=5958) 2024-11-13T18:26:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741928_1107 (size=5958) 2024-11-13T18:26:06,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741928_1107 (size=5958) 2024-11-13T18:26:06,848 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/9520bc72859f46939319585067198449 2024-11-13T18:26:06,853 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/9520bc72859f46939319585067198449 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/9520bc72859f46939319585067198449 2024-11-13T18:26:06,858 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/9520bc72859f46939319585067198449, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T18:26:06,859 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 64ad6e58a5d351a0e777ed7ed62ea43e in 36ms, sequenceid=13, compaction requested=false 2024-11-13T18:26:06,859 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 64ad6e58a5d351a0e777ed7ed62ea43e: 2024-11-13T18:26:06,894 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64ad6e58a5d351a0e777ed7ed62ea43e, disabling compactions & flushes 2024-11-13T18:26:06,894 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,894 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,894 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. after waiting 0 ms 2024-11-13T18:26:06,895 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,896 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,896 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:06,896 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Waiting for close lock at 1731522366894Disabling compacts and flushes for region at 1731522366894Disabling writes for close at 1731522366895 (+1 ms)Writing region close event to WAL at 1731522366896 (+1 ms)Closed at 1731522366896 2024-11-13T18:26:06,899 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 not finished, retry = 0 2024-11-13T18:26:06,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741927_1106 (size=5869) 2024-11-13T18:26:06,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741927_1106 (size=5869) 2024-11-13T18:26:06,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741927_1106 (size=5869) 2024-11-13T18:26:07,016 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, size=5.7 K (5869bytes) 2024-11-13T18:26:07,016 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 2024-11-13T18:26:07,017 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 after 1ms 2024-11-13T18:26:07,019 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 took 3ms 2024-11-13T18:26:07,022 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 so closing down 2024-11-13T18:26:07,022 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:26:07,023 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731522366784.temp 2024-11-13T18:26:07,025 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp 2024-11-13T18:26:07,025 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:26:07,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741929_1108 (size=5223) 2024-11-13T18:26:07,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741929_1108 (size=5223) 2024-11-13T18:26:07,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741929_1108 (size=5223) 2024-11-13T18:26:07,044 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T18:26:07,048 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 2024-11-13T18:26:07,048 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-13T18:26:07,048 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, journal: Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, size=5.7 K (5869bytes) at 1731522367016Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 so closing down at 1731522367022 (+6 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp at 1731522367025 (+3 ms)3 split writer threads finished at 1731522367025Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731522367045 (+20 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000003-wal.1731522366784.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 at 1731522367048 (+3 ms)Processed 32 edits across 1 Regions in 29 ms; skipped=2; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731522367048 2024-11-13T18:26:07,050 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522366784 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522366784 2024-11-13T18:26:07,050 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 2024-11-13T18:26:07,051 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:07,052 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:07,067 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, exclude list is [], retry=0 2024-11-13T18:26:07,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:07,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:07,070 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:07,072 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 2024-11-13T18:26:07,072 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527),(127.0.0.1/127.0.0.1:44725:44725)] 2024-11-13T18:26:07,072 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 64ad6e58a5d351a0e777ed7ed62ea43e, NAME => 'testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.', STARTKEY => '', ENDKEY => ''} 2024-11-13T18:26:07,072 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:07,072 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,072 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,076 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,076 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName a 2024-11-13T18:26:07,077 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:07,081 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/9520bc72859f46939319585067198449 2024-11-13T18:26:07,082 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:07,082 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,082 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName b 2024-11-13T18:26:07,082 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:07,083 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:07,083 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,083 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName c 2024-11-13T18:26:07,083 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:07,084 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:07,084 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,084 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,086 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,086 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 2024-11-13T18:26:07,088 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 2024-11-13T18:26:07,089 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64ad6e58a5d351a0e777ed7ed62ea43e 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T18:26:07,108 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/897f111ad20a4c7f9fd24976da3b6dcf is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731522366859/Put/seqid=0 2024-11-13T18:26:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741931_1110 (size=5958) 2024-11-13T18:26:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741931_1110 (size=5958) 2024-11-13T18:26:07,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741931_1110 (size=5958) 2024-11-13T18:26:07,117 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/897f111ad20a4c7f9fd24976da3b6dcf 2024-11-13T18:26:07,141 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/cda89a05436d4b739513be61d404f26b is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731522366869/Put/seqid=0 2024-11-13T18:26:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741932_1111 (size=5958) 2024-11-13T18:26:07,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741932_1111 (size=5958) 2024-11-13T18:26:07,159 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/cda89a05436d4b739513be61d404f26b 2024-11-13T18:26:07,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741932_1111 (size=5958) 2024-11-13T18:26:07,166 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/897f111ad20a4c7f9fd24976da3b6dcf as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/b/897f111ad20a4c7f9fd24976da3b6dcf 2024-11-13T18:26:07,172 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/b/897f111ad20a4c7f9fd24976da3b6dcf, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T18:26:07,174 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/cda89a05436d4b739513be61d404f26b as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/c/cda89a05436d4b739513be61d404f26b 2024-11-13T18:26:07,180 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/c/cda89a05436d4b739513be61d404f26b, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T18:26:07,180 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 64ad6e58a5d351a0e777ed7ed62ea43e in 92ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T18:26:07,181 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000035 2024-11-13T18:26:07,187 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,187 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,187 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:07,189 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:07,193 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T18:26:07,194 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64ad6e58a5d351a0e777ed7ed62ea43e; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64113470, jitterRate=-0.044634848833084106}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:07,195 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Writing region info on filesystem at 1731522367072Initializing all the Stores at 1731522367073 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522367073Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522367075 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522367076 (+1 ms)Obtaining lock to block concurrent updates at 1731522367089 (+13 ms)Preparing flush snapshotting stores in 64ad6e58a5d351a0e777ed7ed62ea43e at 1731522367089Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731522367089Flushing stores of testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. at 1731522367089Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: creating writer at 1731522367089Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: appending metadata at 1731522367107 (+18 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: closing flushed file at 1731522367107Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: creating writer at 1731522367123 (+16 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: appending metadata at 1731522367141 (+18 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: closing flushed file at 1731522367141Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@8d9bf7c: reopening flushed file at 1731522367165 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@75d16a22: reopening flushed file at 1731522367172 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 64ad6e58a5d351a0e777ed7ed62ea43e in 92ms, sequenceid=35, compaction requested=false; wal=null at 1731522367180 (+8 ms)Cleaning up temporary data from old regions at 1731522367187 (+7 ms)Region opened successfully at 1731522367195 (+8 ms) 2024-11-13T18:26:07,290 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, size=0 (0bytes) 2024-11-13T18:26:07,290 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 might be still open, length is 0 2024-11-13T18:26:07,290 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 2024-11-13T18:26:07,290 WARN [IPC Server handler 0 on default port 40367 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-11-13T18:26:07,291 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 after 1ms 2024-11-13T18:26:07,878 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T18:26:07,878 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T18:26:07,879 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T18:26:07,879 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T18:26:08,229 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:59124 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:33523:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59124 dst: /127.0.0.1:33523 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:33523 remote=/127.0.0.1:59124]. Total timeout mills is 60000, 59013 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:08,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:33358 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:46817:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33358 dst: /127.0.0.1:46817 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:08,230 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1035578202_22 at /127.0.0.1:38772 [Receiving block BP-1832393325-172.17.0.3-1731522323869:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:41041:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38772 dst: /127.0.0.1:41041 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:08,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741930_1112 (size=5215) 2024-11-13T18:26:08,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741930_1112 (size=5215) 2024-11-13T18:26:08,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741930_1112 (size=5215) 2024-11-13T18:26:11,292 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 after 4002ms 2024-11-13T18:26:11,295 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 took 4005ms 2024-11-13T18:26:11,296 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052; continuing. 2024-11-13T18:26:11,296 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 so closing down 2024-11-13T18:26:11,297 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T18:26:11,298 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731522367052.temp 2024-11-13T18:26:11,299 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp 2024-11-13T18:26:11,299 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T18:26:11,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741933_1113 (size=5223) 2024-11-13T18:26:11,306 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T18:26:11,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741933_1113 (size=5223) 2024-11-13T18:26:11,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741933_1113 (size=5223) 2024-11-13T18:26:11,307 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 2024-11-13T18:26:11,307 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T18:26:11,308 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, journal: Splitting hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, size=0 (0bytes) at 1731522367290Finishing writing output for hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 so closing down at 1731522371296 (+4006 ms)Creating recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp at 1731522371299 (+3 ms)3 split writer threads finished at 1731522371299Closed recovered edits writer path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731522371306 (+7 ms)Rename recovered edits hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000037-wal.1731522367052.temp to hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 at 1731522371307 (+1 ms)Processed 30 edits across 1 Regions in 12 ms; skipped=0; WAL=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052, size=0, length=0, corrupted=false, cancelled=false at 1731522371307 2024-11-13T18:26:11,309 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 to hdfs://localhost:40367/hbase/oldWALs/wal.1731522367052 2024-11-13T18:26:11,310 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 2024-11-13T18:26:11,310 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T18:26:11,312 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:40367/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626, archiveDir=hdfs://localhost:40367/hbase/oldWALs, maxLogs=32 2024-11-13T18:26:11,327 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522371313, exclude list is [], retry=0 2024-11-13T18:26:11,329 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:46817,DS-113ec9e2-4543-4828-b945-f3ecb16f5df1,DISK] 2024-11-13T18:26:11,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:33523,DS-9768edff-b9ba-44c8-ab76-6ac67e41de83,DISK] 2024-11-13T18:26:11,330 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41041,DS-36951baa-cfb8-4898-850b-e4f621da6010,DISK] 2024-11-13T18:26:11,331 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522371313 2024-11-13T18:26:11,332 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44725:44725),(127.0.0.1/127.0.0.1:36085:36085),(127.0.0.1/127.0.0.1:41527:41527)] 2024-11-13T18:26:11,332 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T18:26:11,333 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,334 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName a 2024-11-13T18:26:11,334 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:11,339 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/9520bc72859f46939319585067198449 2024-11-13T18:26:11,339 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:11,339 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,340 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName b 2024-11-13T18:26:11,340 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:11,346 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/b/897f111ad20a4c7f9fd24976da3b6dcf 2024-11-13T18:26:11,346 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:11,346 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,347 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 64ad6e58a5d351a0e777ed7ed62ea43e columnFamilyName c 2024-11-13T18:26:11,347 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T18:26:11,351 DEBUG [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/c/cda89a05436d4b739513be61d404f26b 2024-11-13T18:26:11,351 INFO [StoreOpener-64ad6e58a5d351a0e777ed7ed62ea43e-1 {}] regionserver.HStore(327): Store=64ad6e58a5d351a0e777ed7ed62ea43e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T18:26:11,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,352 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,354 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 2024-11-13T18:26:11,360 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 2024-11-13T18:26:11,360 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 64ad6e58a5d351a0e777ed7ed62ea43e 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T18:26:11,406 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/ecb3571206e2478b84cbf7855a0d6376 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731522367204/Put/seqid=0 2024-11-13T18:26:11,410 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-13T18:26:11,410 INFO [master/fda96cf025dc:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-13T18:26:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741935_1115 (size=5958) 2024-11-13T18:26:11,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741935_1115 (size=5958) 2024-11-13T18:26:11,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741935_1115 (size=5958) 2024-11-13T18:26:11,417 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/ecb3571206e2478b84cbf7855a0d6376 2024-11-13T18:26:11,439 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/3c99fb9323fc4005be3f9e3481f42834 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731522367211/Put/seqid=0 2024-11-13T18:26:11,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741936_1116 (size=5958) 2024-11-13T18:26:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741936_1116 (size=5958) 2024-11-13T18:26:11,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741936_1116 (size=5958) 2024-11-13T18:26:11,446 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/3c99fb9323fc4005be3f9e3481f42834 2024-11-13T18:26:11,466 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/d4ec4e20d4a746f19335c044c744b6a8 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731522367228/Put/seqid=0 2024-11-13T18:26:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741937_1117 (size=5958) 2024-11-13T18:26:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741937_1117 (size=5958) 2024-11-13T18:26:11,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741937_1117 (size=5958) 2024-11-13T18:26:11,477 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/d4ec4e20d4a746f19335c044c744b6a8 2024-11-13T18:26:11,483 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/a/ecb3571206e2478b84cbf7855a0d6376 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/ecb3571206e2478b84cbf7855a0d6376 2024-11-13T18:26:11,489 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/a/ecb3571206e2478b84cbf7855a0d6376, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T18:26:11,490 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/b/3c99fb9323fc4005be3f9e3481f42834 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/b/3c99fb9323fc4005be3f9e3481f42834 2024-11-13T18:26:11,494 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/b/3c99fb9323fc4005be3f9e3481f42834, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T18:26:11,495 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/.tmp/c/d4ec4e20d4a746f19335c044c744b6a8 as hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/c/d4ec4e20d4a746f19335c044c744b6a8 2024-11-13T18:26:11,501 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/c/d4ec4e20d4a746f19335c044c744b6a8, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T18:26:11,501 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 64ad6e58a5d351a0e777ed7ed62ea43e in 141ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T18:26:11,502 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/0000000000000000066 2024-11-13T18:26:11,503 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,503 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,504 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T18:26:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 64ad6e58a5d351a0e777ed7ed62ea43e 2024-11-13T18:26:11,508 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/hbase/data/default/testReplayEditsWrittenViaHRegion/64ad6e58a5d351a0e777ed7ed62ea43e/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T18:26:11,509 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 64ad6e58a5d351a0e777ed7ed62ea43e; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62186131, jitterRate=-0.07335443794727325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T18:26:11,509 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Writing region info on filesystem at 1731522371332Initializing all the Stores at 1731522371333 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522371333Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522371333Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731522371333Obtaining lock to block concurrent updates at 1731522371360 (+27 ms)Preparing flush snapshotting stores in 64ad6e58a5d351a0e777ed7ed62ea43e at 1731522371360Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731522371361 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. at 1731522371361Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/a: creating writer at 1731522371361Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/a: appending metadata at 1731522371405 (+44 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/a: closing flushed file at 1731522371405Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: creating writer at 1731522371423 (+18 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: appending metadata at 1731522371439 (+16 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/b: closing flushed file at 1731522371439Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: creating writer at 1731522371451 (+12 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: appending metadata at 1731522371465 (+14 ms)Flushing 64ad6e58a5d351a0e777ed7ed62ea43e/c: closing flushed file at 1731522371465Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77e0e946: reopening flushed file at 1731522371482 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2e30a538: reopening flushed file at 1731522371489 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c504d99: reopening flushed file at 1731522371495 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 64ad6e58a5d351a0e777ed7ed62ea43e in 141ms, sequenceid=66, compaction requested=false; wal=null at 1731522371501 (+6 ms)Cleaning up temporary data from old regions at 1731522371503 (+2 ms)Region opened successfully at 1731522371509 (+6 ms) 2024-11-13T18:26:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 64ad6e58a5d351a0e777ed7ed62ea43e, disabling compactions & flushes 2024-11-13T18:26:11,527 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. after waiting 0 ms 2024-11-13T18:26:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:11,531 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731522366629.64ad6e58a5d351a0e777ed7ed62ea43e. 2024-11-13T18:26:11,531 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 64ad6e58a5d351a0e777ed7ed62ea43e: Waiting for close lock at 1731522371527Disabling compacts and flushes for region at 1731522371527Disabling writes for close at 1731522371527Writing region close event to WAL at 1731522371531 (+4 ms)Closed at 1731522371531 2024-11-13T18:26:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741934_1114 (size=93) 2024-11-13T18:26:11,537 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522371313 not finished, retry = 0 2024-11-13T18:26:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741934_1114 (size=93) 2024-11-13T18:26:11,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741934_1114 (size=93) 2024-11-13T18:26:11,639 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T18:26:11,639 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731522371313) 2024-11-13T18:26:11,656 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=440 (was 439) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:40367 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:40367 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261547888_22 at /127.0.0.1:33404 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261547888_22 at /127.0.0.1:38786 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261547888_22 at /127.0.0.1:59176 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1410 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=509 (was 510), ProcessCount=11 (was 11), AvailableMemoryMB=1840 (was 2068) 2024-11-13T18:26:11,656 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1410 is superior to 1024 2024-11-13T18:26:11,656 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T18:26:11,656 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:11,657 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:11,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,657 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,657 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T18:26:11,658 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T18:26:11,658 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1949787305, stopped=false 2024-11-13T18:26:11,658 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fda96cf025dc,42825,1731522327393 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T18:26:11,660 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:11,660 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:26:11,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:26:11,661 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T18:26:11,661 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:11,661 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:26:11,661 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T18:26:11,661 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,33531,1731522328314' ***** 2024-11-13T18:26:11,661 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:26:11,662 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fda96cf025dc,44035,1731522328435' ***** 2024-11-13T18:26:11,662 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T18:26:11,662 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:26:11,662 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T18:26:11,662 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:26:11,662 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T18:26:11,662 INFO [RS:0;fda96cf025dc:33531 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:26:11,662 INFO [RS:0;fda96cf025dc:33531 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:26:11,663 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,33531,1731522328314 2024-11-13T18:26:11,663 INFO [RS:0;fda96cf025dc:33531 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T18:26:11,663 INFO [RS:0;fda96cf025dc:33531 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fda96cf025dc:33531. 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T18:26:11,663 DEBUG [RS:0;fda96cf025dc:33531 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:11,663 DEBUG [RS:0;fda96cf025dc:33531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 99991ec418f4188651e0104287d02355 2024-11-13T18:26:11,663 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,33531,1731522328314; all regions closed. 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(959): stopping server fda96cf025dc,44035,1731522328435 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:26:11,663 INFO [RS:1;fda96cf025dc:44035 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;fda96cf025dc:44035. 2024-11-13T18:26:11,663 DEBUG [RS:1;fda96cf025dc:44035 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T18:26:11,663 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 99991ec418f4188651e0104287d02355, disabling compactions & flushes 2024-11-13T18:26:11,664 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:26:11,664 DEBUG [RS:1;fda96cf025dc:44035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,664 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:26:11,664 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. after waiting 0 ms 2024-11-13T18:26:11,664 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:26:11,664 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:26:11,664 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:26:11,664 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:26:11,664 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T18:26:11,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741835_1011 (size=2598) 2024-11-13T18:26:11,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741835_1011 (size=2598) 2024-11-13T18:26:11,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741835_1011 (size=2598) 2024-11-13T18:26:11,668 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T18:26:11,668 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1325): Online Regions={99991ec418f4188651e0104287d02355=testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355., 1588230740=hbase:meta,,1.1588230740} 2024-11-13T18:26:11,669 DEBUG [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 99991ec418f4188651e0104287d02355 2024-11-13T18:26:11,669 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T18:26:11,669 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T18:26:11,669 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T18:26:11,669 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T18:26:11,669 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T18:26:11,669 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-11-13T18:26:11,672 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:11,672 DEBUG [RS:0;fda96cf025dc:33531 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C33531%2C1731522328314:(num 1731522330201) 2024-11-13T18:26:11,673 DEBUG [RS:0;fda96cf025dc:33531 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T18:26:11,673 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T18:26:11,673 INFO [RS:0;fda96cf025dc:33531 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:26:11,674 INFO [RS:0;fda96cf025dc:33531 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:33531 2024-11-13T18:26:11,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:26:11,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,33531,1731522328314 2024-11-13T18:26:11,675 INFO [RS:0;fda96cf025dc:33531 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:26:11,676 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/default/testReplayEditsAfterRegionMovedWithMultiCF/99991ec418f4188651e0104287d02355/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T18:26:11,676 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,33531,1731522328314] 2024-11-13T18:26:11,677 INFO [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:26:11,677 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 99991ec418f4188651e0104287d02355: Waiting for close lock at 1731522371663Running coprocessor pre-close hooks at 1731522371663Disabling compacts and flushes for region at 1731522371663Disabling writes for close at 1731522371664 (+1 ms)Writing region close event to WAL at 1731522371670 (+6 ms)Running coprocessor post-close hooks at 1731522371677 (+7 ms)Closed at 1731522371677 2024-11-13T18:26:11,677 DEBUG [RS_CLOSE_REGION-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355. 2024-11-13T18:26:11,679 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,33531,1731522328314 already deleted, retry=false 2024-11-13T18:26:11,679 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,33531,1731522328314 expired; onlineServers=1 2024-11-13T18:26:11,702 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/info/6aaa4470f14d4e839b8ca36ade543e1d is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731522346370.99991ec418f4188651e0104287d02355./info:regioninfo/1731522349920/Put/seqid=0 2024-11-13T18:26:11,705 INFO [regionserver/fda96cf025dc:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:11,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741938_1118 (size=8243) 2024-11-13T18:26:11,712 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741938_1118 (size=8243) 2024-11-13T18:26:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741938_1118 (size=8243) 2024-11-13T18:26:11,714 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/info/6aaa4470f14d4e839b8ca36ade543e1d 2024-11-13T18:26:11,735 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/ns/44e7ebb26765435bb9a5b12b3b63b304 is 43, key is default/ns:d/1731522330784/Put/seqid=0 2024-11-13T18:26:11,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741939_1119 (size=5153) 2024-11-13T18:26:11,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741939_1119 (size=5153) 2024-11-13T18:26:11,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741939_1119 (size=5153) 2024-11-13T18:26:11,742 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/ns/44e7ebb26765435bb9a5b12b3b63b304 2024-11-13T18:26:11,768 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/table/b79ceb15391f4846852bebd1f8967b84 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731522346790/Put/seqid=0 2024-11-13T18:26:11,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741940_1120 (size=5431) 2024-11-13T18:26:11,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741940_1120 (size=5431) 2024-11-13T18:26:11,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741940_1120 (size=5431) 2024-11-13T18:26:11,777 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/table/b79ceb15391f4846852bebd1f8967b84 2024-11-13T18:26:11,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:11,777 INFO [RS:0;fda96cf025dc:33531 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:26:11,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:33531-0x1003a6402a40001, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:11,777 INFO [RS:0;fda96cf025dc:33531 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,33531,1731522328314; zookeeper connection closed. 2024-11-13T18:26:11,778 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@de7c0f3 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@de7c0f3 2024-11-13T18:26:11,783 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/info/6aaa4470f14d4e839b8ca36ade543e1d as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/info/6aaa4470f14d4e839b8ca36ade543e1d 2024-11-13T18:26:11,787 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/info/6aaa4470f14d4e839b8ca36ade543e1d, entries=18, sequenceid=21, filesize=8.0 K 2024-11-13T18:26:11,788 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/ns/44e7ebb26765435bb9a5b12b3b63b304 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/ns/44e7ebb26765435bb9a5b12b3b63b304 2024-11-13T18:26:11,792 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/ns/44e7ebb26765435bb9a5b12b3b63b304, entries=2, sequenceid=21, filesize=5.0 K 2024-11-13T18:26:11,793 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/.tmp/table/b79ceb15391f4846852bebd1f8967b84 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/table/b79ceb15391f4846852bebd1f8967b84 2024-11-13T18:26:11,797 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/table/b79ceb15391f4846852bebd1f8967b84, entries=2, sequenceid=21, filesize=5.3 K 2024-11-13T18:26:11,799 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=21, compaction requested=false 2024-11-13T18:26:11,803 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-11-13T18:26:11,804 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T18:26:11,804 INFO [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T18:26:11,805 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731522371668Running coprocessor pre-close hooks at 1731522371668Disabling compacts and flushes for region at 1731522371668Disabling writes for close at 1731522371669 (+1 ms)Obtaining lock to block concurrent updates at 1731522371669Preparing flush snapshotting stores in 1588230740 at 1731522371669Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1731522371669Flushing stores of hbase:meta,,1.1588230740 at 1731522371671 (+2 ms)Flushing 1588230740/info: creating writer at 1731522371671Flushing 1588230740/info: appending metadata at 1731522371702 (+31 ms)Flushing 1588230740/info: closing flushed file at 1731522371702Flushing 1588230740/ns: creating writer at 1731522371720 (+18 ms)Flushing 1588230740/ns: appending metadata at 1731522371734 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1731522371734Flushing 1588230740/table: creating writer at 1731522371747 (+13 ms)Flushing 1588230740/table: appending metadata at 1731522371768 (+21 ms)Flushing 1588230740/table: closing flushed file at 1731522371768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15dd7484: reopening flushed file at 1731522371782 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@211f38bf: reopening flushed file at 1731522371787 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e8fdd4d: reopening flushed file at 1731522371793 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 130ms, sequenceid=21, compaction requested=false at 1731522371799 (+6 ms)Writing region close event to WAL at 1731522371800 (+1 ms)Running coprocessor post-close hooks at 1731522371804 (+4 ms)Closed at 1731522371804 2024-11-13T18:26:11,805 DEBUG [RS_CLOSE_META-regionserver/fda96cf025dc:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T18:26:11,869 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(976): stopping server fda96cf025dc,44035,1731522328435; all regions closed. 2024-11-13T18:26:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741836_1012 (size=8899) 2024-11-13T18:26:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741836_1012 (size=8899) 2024-11-13T18:26:11,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741836_1012 (size=8899) 2024-11-13T18:26:11,874 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.meta.1731522330599.meta not finished, retry = 0 2024-11-13T18:26:11,977 DEBUG [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs 2024-11-13T18:26:11,977 INFO [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C44035%2C1731522328435.meta:.meta(num 1731522330599) 2024-11-13T18:26:11,981 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/WALs/fda96cf025dc,44035,1731522328435/fda96cf025dc%2C44035%2C1731522328435.1731522330187 not finished, retry = 0 2024-11-13T18:26:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741833_1009 (size=928) 2024-11-13T18:26:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741833_1009 (size=928) 2024-11-13T18:26:11,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741833_1009 (size=928) 2024-11-13T18:26:12,027 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-13T18:26:12,027 INFO [regionserver/fda96cf025dc:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-13T18:26:12,084 DEBUG [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/oldWALs 2024-11-13T18:26:12,084 INFO [RS:1;fda96cf025dc:44035 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fda96cf025dc%2C44035%2C1731522328435:(num 1731522330187) 2024-11-13T18:26:12,084 DEBUG [RS:1;fda96cf025dc:44035 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T18:26:12,084 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T18:26:12,084 INFO [RS:1;fda96cf025dc:44035 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:26:12,084 INFO [RS:1;fda96cf025dc:44035 {}] hbase.ChoreService(370): Chore service for: regionserver/fda96cf025dc:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T18:26:12,085 INFO [RS:1;fda96cf025dc:44035 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:26:12,085 INFO [regionserver/fda96cf025dc:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:26:12,085 INFO [RS:1;fda96cf025dc:44035 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:44035 2024-11-13T18:26:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fda96cf025dc,44035,1731522328435 2024-11-13T18:26:12,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T18:26:12,088 INFO [RS:1;fda96cf025dc:44035 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:26:12,089 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fda96cf025dc,44035,1731522328435] 2024-11-13T18:26:12,091 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fda96cf025dc,44035,1731522328435 already deleted, retry=false 2024-11-13T18:26:12,091 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fda96cf025dc,44035,1731522328435 expired; onlineServers=0 2024-11-13T18:26:12,091 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fda96cf025dc,42825,1731522327393' ***** 2024-11-13T18:26:12,091 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T18:26:12,091 INFO [M:0;fda96cf025dc:42825 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T18:26:12,091 INFO [M:0;fda96cf025dc:42825 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T18:26:12,091 DEBUG [M:0;fda96cf025dc:42825 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T18:26:12,092 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T18:26:12,092 DEBUG [M:0;fda96cf025dc:42825 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T18:26:12,092 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522329741 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.large.0-1731522329741,5,FailOnTimeoutGroup] 2024-11-13T18:26:12,092 INFO [M:0;fda96cf025dc:42825 {}] hbase.ChoreService(370): Chore service for: master/fda96cf025dc:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T18:26:12,092 INFO [M:0;fda96cf025dc:42825 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T18:26:12,092 DEBUG [master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522329745 {}] cleaner.HFileCleaner(306): Exit Thread[master/fda96cf025dc:0:becomeActiveMaster-HFileCleaner.small.0-1731522329745,5,FailOnTimeoutGroup] 2024-11-13T18:26:12,092 DEBUG [M:0;fda96cf025dc:42825 {}] master.HMaster(1795): Stopping service threads 2024-11-13T18:26:12,092 INFO [M:0;fda96cf025dc:42825 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T18:26:12,092 INFO [M:0;fda96cf025dc:42825 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T18:26:12,093 INFO [M:0;fda96cf025dc:42825 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T18:26:12,093 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T18:26:12,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T18:26:12,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T18:26:12,094 DEBUG [M:0;fda96cf025dc:42825 {}] zookeeper.ZKUtil(347): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T18:26:12,094 WARN [M:0;fda96cf025dc:42825 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T18:26:12,095 INFO [M:0;fda96cf025dc:42825 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/.lastflushedseqids 2024-11-13T18:26:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741941_1121 (size=138) 2024-11-13T18:26:12,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741941_1121 (size=138) 2024-11-13T18:26:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741941_1121 (size=138) 2024-11-13T18:26:12,126 INFO [M:0;fda96cf025dc:42825 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T18:26:12,126 INFO [M:0;fda96cf025dc:42825 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T18:26:12,126 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T18:26:12,126 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:12,126 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:12,126 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T18:26:12,126 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:12,126 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.34 KB heapSize=83.73 KB 2024-11-13T18:26:12,145 DEBUG [M:0;fda96cf025dc:42825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9edfc909d58b41a7b33760b40dbcb6d1 is 82, key is hbase:meta,,1/info:regioninfo/1731522330684/Put/seqid=0 2024-11-13T18:26:12,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741942_1122 (size=5672) 2024-11-13T18:26:12,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741942_1122 (size=5672) 2024-11-13T18:26:12,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741942_1122 (size=5672) 2024-11-13T18:26:12,157 INFO [M:0;fda96cf025dc:42825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9edfc909d58b41a7b33760b40dbcb6d1 2024-11-13T18:26:12,183 DEBUG [M:0;fda96cf025dc:42825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34eccbccc59b40ba8771687a3609196b is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731522346796/Put/seqid=0 2024-11-13T18:26:12,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:12,189 INFO [RS:1;fda96cf025dc:44035 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:26:12,189 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:44035-0x1003a6402a40002, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:12,189 INFO [RS:1;fda96cf025dc:44035 {}] regionserver.HRegionServer(1031): Exiting; stopping=fda96cf025dc,44035,1731522328435; zookeeper connection closed. 2024-11-13T18:26:12,193 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@67d2a210 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@67d2a210 2024-11-13T18:26:12,193 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T18:26:12,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741943_1123 (size=7755) 2024-11-13T18:26:12,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741943_1123 (size=7755) 2024-11-13T18:26:12,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741943_1123 (size=7755) 2024-11-13T18:26:12,197 INFO [M:0;fda96cf025dc:42825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.61 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34eccbccc59b40ba8771687a3609196b 2024-11-13T18:26:12,202 INFO [M:0;fda96cf025dc:42825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34eccbccc59b40ba8771687a3609196b 2024-11-13T18:26:12,225 DEBUG [M:0;fda96cf025dc:42825 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a91c9560ef11456db976b9fd8a623c97 is 69, key is fda96cf025dc,33531,1731522328314/rs:state/1731522329808/Put/seqid=0 2024-11-13T18:26:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741944_1124 (size=5445) 2024-11-13T18:26:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741944_1124 (size=5445) 2024-11-13T18:26:12,243 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741944_1124 (size=5445) 2024-11-13T18:26:12,244 INFO [M:0;fda96cf025dc:42825 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a91c9560ef11456db976b9fd8a623c97 2024-11-13T18:26:12,249 INFO [M:0;fda96cf025dc:42825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a91c9560ef11456db976b9fd8a623c97 2024-11-13T18:26:12,249 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/9edfc909d58b41a7b33760b40dbcb6d1 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9edfc909d58b41a7b33760b40dbcb6d1 2024-11-13T18:26:12,254 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/9edfc909d58b41a7b33760b40dbcb6d1, entries=8, sequenceid=168, filesize=5.5 K 2024-11-13T18:26:12,254 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/34eccbccc59b40ba8771687a3609196b as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/34eccbccc59b40ba8771687a3609196b 2024-11-13T18:26:12,258 INFO [M:0;fda96cf025dc:42825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 34eccbccc59b40ba8771687a3609196b 2024-11-13T18:26:12,259 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/34eccbccc59b40ba8771687a3609196b, entries=17, sequenceid=168, filesize=7.6 K 2024-11-13T18:26:12,259 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a91c9560ef11456db976b9fd8a623c97 as hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a91c9560ef11456db976b9fd8a623c97 2024-11-13T18:26:12,264 INFO [M:0;fda96cf025dc:42825 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a91c9560ef11456db976b9fd8a623c97 2024-11-13T18:26:12,264 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40367/user/jenkins/test-data/631c94f1-db28-2ab8-5b52-3d2db09247d4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a91c9560ef11456db976b9fd8a623c97, entries=3, sequenceid=168, filesize=5.3 K 2024-11-13T18:26:12,265 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=168, compaction requested=false 2024-11-13T18:26:12,266 INFO [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T18:26:12,266 DEBUG [M:0;fda96cf025dc:42825 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731522372126Disabling compacts and flushes for region at 1731522372126Disabling writes for close at 1731522372126Obtaining lock to block concurrent updates at 1731522372126Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731522372126Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69984, getHeapSize=85672, getOffHeapSize=0, getCellsCount=195 at 1731522372127 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731522372128 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731522372128Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731522372145 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731522372145Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731522372162 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731522372182 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731522372182Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731522372202 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731522372224 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731522372224Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cd48bc4: reopening flushed file at 1731522372249 (+25 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@76632a6a: reopening flushed file at 1731522372254 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38bf6377: reopening flushed file at 1731522372259 (+5 ms)Finished flush of dataSize ~68.34 KB/69984, heapSize ~83.43 KB/85432, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 139ms, sequenceid=168, compaction requested=false at 1731522372265 (+6 ms)Writing region close event to WAL at 1731522372266 (+1 ms)Closed at 1731522372266 2024-11-13T18:26:12,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:46817 is added to blk_1073741830_1006 (size=81431) 2024-11-13T18:26:12,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41041 is added to blk_1073741830_1006 (size=81431) 2024-11-13T18:26:12,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:33523 is added to blk_1073741830_1006 (size=81431) 2024-11-13T18:26:12,270 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T18:26:12,270 INFO [M:0;fda96cf025dc:42825 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T18:26:12,270 INFO [M:0;fda96cf025dc:42825 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42825 2024-11-13T18:26:12,271 INFO [M:0;fda96cf025dc:42825 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T18:26:12,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:12,373 INFO [M:0;fda96cf025dc:42825 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T18:26:12,373 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:42825-0x1003a6402a40000, quorum=127.0.0.1:54860, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T18:26:12,379 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350722 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350722 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731522350324/wal.1731522350722 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T18:26:12,380 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731522341457/wal.1731522342044 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:12,381 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731522366626/wal.1731522367052 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:12,383 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522360127 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522360127 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731522350865/wal.1731522360127 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T18:26:12,383 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731522332532/wal.1731522332608 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:12,386 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522361025 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522361025 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731522360389/wal.1731522361025 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T18:26:12,387 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731522361305/wal.1731522361841 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T18:26:12,389 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal.1731522332356 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal.1731522332356 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731522331728/wal.1731522332356 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T18:26:12,392 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal.1731522331358 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal.1731522331358 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731522331056/wal.1731522331358 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1035578202_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 25 more 2024-11-13T18:26:12,406 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:26:12,409 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:12,409 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:12,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:12,410 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:12,414 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:26:12,414 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1832393325-172.17.0.3-1731522323869 (Datanode Uuid af374aac-32d3-409a-b3c5-fa6d863690b8) service to localhost/127.0.0.1:40367 2024-11-13T18:26:12,416 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data5/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,416 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data6/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,418 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:26:12,418 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:26:12,418 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:26:12,425 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:26:12,426 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:12,426 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:12,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:12,426 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:12,428 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:26:12,428 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:26:12,428 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1832393325-172.17.0.3-1731522323869 (Datanode Uuid bf437f33-3b70-4d83-a1a4-46e973f1f052) service to localhost/127.0.0.1:40367 2024-11-13T18:26:12,428 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:26:12,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data3/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,429 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data4/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,429 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:26:12,450 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T18:26:12,450 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:12,450 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:12,450 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:12,451 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:12,453 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T18:26:12,453 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T18:26:12,456 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T18:26:12,456 WARN [BP-1832393325-172.17.0.3-1731522323869 heartbeating to localhost/127.0.0.1:40367 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1832393325-172.17.0.3-1731522323869 (Datanode Uuid 1b19a530-625c-4e79-b632-9290c2b4e239) service to localhost/127.0.0.1:40367 2024-11-13T18:26:12,458 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data2/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,458 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/cluster_0aed6a03-c82e-8ab1-6002-e50a1bf40a88/data/data1/current/BP-1832393325-172.17.0.3-1731522323869 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T18:26:12,458 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T18:26:12,469 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T18:26:12,470 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T18:26:12,470 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T18:26:12,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T18:26:12,470 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/d2e2e8fe-2486-5e35-19fa-7edf107b32b2/hadoop.log.dir/,STOPPED} 2024-11-13T18:26:12,487 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T18:26:12,568 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down