2024-12-04 20:16:38,275 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 20:16:38,286 main DEBUG Took 0.009037 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-04 20:16:38,286 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-04 20:16:38,287 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-04 20:16:38,288 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-04 20:16:38,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,295 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-04 20:16:38,306 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,307 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,308 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,308 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,309 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,309 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,310 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,310 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,311 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,311 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,312 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,312 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,312 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,313 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,313 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,314 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,314 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,314 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,315 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,315 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,316 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-04 20:16:38,316 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,316 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-04 20:16:38,318 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-04 20:16:38,319 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-04 20:16:38,320 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-04 20:16:38,321 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-04 20:16:38,322 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-04 20:16:38,322 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-04 20:16:38,330 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-04 20:16:38,333 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-04 20:16:38,334 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-04 20:16:38,334 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-04 20:16:38,335 main DEBUG createAppenders(={Console}) 2024-12-04 20:16:38,335 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-04 20:16:38,336 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-04 20:16:38,336 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-04 20:16:38,336 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-04 20:16:38,337 main DEBUG OutputStream closed 2024-12-04 20:16:38,337 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-04 20:16:38,337 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-04 20:16:38,337 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-04 20:16:38,401 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-04 20:16:38,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-04 20:16:38,403 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-04 20:16:38,404 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-04 20:16:38,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-04 20:16:38,405 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-04 20:16:38,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-04 20:16:38,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-04 20:16:38,406 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-04 20:16:38,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-04 20:16:38,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-04 20:16:38,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-04 20:16:38,407 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-04 20:16:38,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-04 20:16:38,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-04 20:16:38,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-04 20:16:38,408 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-04 20:16:38,409 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-04 20:16:38,411 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04 20:16:38,411 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-04 20:16:38,412 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-04 20:16:38,412 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-04T20:16:38,652 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2 2024-12-04 20:16:38,655 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-04 20:16:38,655 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-04T20:16:38,662 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-04T20:16:38,668 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-04T20:16:38,690 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-04T20:16:38,734 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-04T20:16:38,735 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-04T20:16:38,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-04T20:16:38,758 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65, deleteOnExit=true 2024-12-04T20:16:38,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-04T20:16:38,758 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/test.cache.data in system properties and HBase conf 2024-12-04T20:16:38,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.tmp.dir in system properties and HBase conf 2024-12-04T20:16:38,759 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir in system properties and HBase conf 2024-12-04T20:16:38,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-04T20:16:38,760 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-04T20:16:38,761 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-04T20:16:38,825 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-04T20:16:38,898 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-04T20:16:38,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:16:38,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-04T20:16:38,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-04T20:16:38,904 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:16:38,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-04T20:16:38,905 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-04T20:16:38,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-04T20:16:38,906 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:16:38,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-04T20:16:38,907 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/nfs.dump.dir in system properties and HBase conf 2024-12-04T20:16:38,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/java.io.tmpdir in system properties and HBase conf 2024-12-04T20:16:38,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-04T20:16:38,908 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-04T20:16:38,909 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-04T20:16:39,577 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-04T20:16:39,644 INFO [Time-limited test {}] log.Log(170): Logging initialized @1953ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-04T20:16:39,711 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:16:39,763 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:16:39,779 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:16:39,780 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:16:39,781 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:16:39,799 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:16:39,801 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:16:39,803 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:16:39,969 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/java.io.tmpdir/jetty-localhost-41431-hadoop-hdfs-3_4_1-tests_jar-_-any-4624709909594598375/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:16:39,974 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:41431} 2024-12-04T20:16:39,974 INFO [Time-limited test {}] server.Server(415): Started @2283ms 2024-12-04T20:16:40,278 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:16:40,285 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:16:40,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:16:40,286 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:16:40,286 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:16:40,287 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:16:40,288 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:16:40,382 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/java.io.tmpdir/jetty-localhost-33031-hadoop-hdfs-3_4_1-tests_jar-_-any-7884511463540800043/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:16:40,383 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:33031} 2024-12-04T20:16:40,384 INFO [Time-limited test {}] server.Server(415): Started @2692ms 2024-12-04T20:16:40,428 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:16:40,548 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:16:40,554 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:16:40,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:16:40,556 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:16:40,556 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-04T20:16:40,557 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:16:40,558 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:16:40,664 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/java.io.tmpdir/jetty-localhost-44695-hadoop-hdfs-3_4_1-tests_jar-_-any-10290325297197063861/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:16:40,665 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:44695} 2024-12-04T20:16:40,665 INFO [Time-limited test {}] server.Server(415): Started @2974ms 2024-12-04T20:16:40,668 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:16:40,705 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-04T20:16:40,709 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-04T20:16:40,716 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-04T20:16:40,716 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-04T20:16:40,717 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-04T20:16:40,718 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,AVAILABLE} 2024-12-04T20:16:40,719 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-04T20:16:40,794 WARN [Thread-105 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data1/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,794 WARN [Thread-107 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data3/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,794 WARN [Thread-108 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data4/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,794 WARN [Thread-106 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data2/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,834 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/java.io.tmpdir/jetty-localhost-44237-hadoop-hdfs-3_4_1-tests_jar-_-any-4833348195037429732/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:16:40,835 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:44237} 2024-12-04T20:16:40,835 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:16:40,835 INFO [Time-limited test {}] server.Server(415): Started @3144ms 2024-12-04T20:16:40,836 WARN [Thread-82 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:16:40,838 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-04T20:16:40,896 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13f26fd9fdcd080e with lease ID 0x3bf1f535b6ceea60: Processing first storage report for DS-b5812768-870d-43ac-b12c-85ccd56a7c36 from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=cbc111ee-df59-4e3c-8cc5-988e06dc269b, infoPort=37929, infoSecurePort=0, ipcPort=37665, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,897 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13f26fd9fdcd080e with lease ID 0x3bf1f535b6ceea60: from storage DS-b5812768-870d-43ac-b12c-85ccd56a7c36 node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=cbc111ee-df59-4e3c-8cc5-988e06dc269b, infoPort=37929, infoSecurePort=0, ipcPort=37665, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-04T20:16:40,897 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea6dbfe4f4c74876 with lease ID 0x3bf1f535b6ceea61: Processing first storage report for DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2 from datanode DatanodeRegistration(127.0.0.1:42009, datanodeUuid=65fdf03d-c3df-49c9-aab8-3a2af9efceef, infoPort=37901, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,897 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea6dbfe4f4c74876 with lease ID 0x3bf1f535b6ceea61: from storage DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2 node DatanodeRegistration(127.0.0.1:42009, datanodeUuid=65fdf03d-c3df-49c9-aab8-3a2af9efceef, infoPort=37901, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:16:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x13f26fd9fdcd080e with lease ID 0x3bf1f535b6ceea60: Processing first storage report for DS-cc812f1c-0bfd-4575-9148-0abfc6086692 from datanode DatanodeRegistration(127.0.0.1:38071, datanodeUuid=cbc111ee-df59-4e3c-8cc5-988e06dc269b, infoPort=37929, infoSecurePort=0, ipcPort=37665, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x13f26fd9fdcd080e with lease ID 0x3bf1f535b6ceea60: from storage DS-cc812f1c-0bfd-4575-9148-0abfc6086692 node DatanodeRegistration(127.0.0.1:38071, datanodeUuid=cbc111ee-df59-4e3c-8cc5-988e06dc269b, infoPort=37929, infoSecurePort=0, ipcPort=37665, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:16:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xea6dbfe4f4c74876 with lease ID 0x3bf1f535b6ceea61: Processing first storage report for DS-3e6a63ea-e623-4088-8abc-9563543f04e9 from datanode DatanodeRegistration(127.0.0.1:42009, datanodeUuid=65fdf03d-c3df-49c9-aab8-3a2af9efceef, infoPort=37901, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,898 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xea6dbfe4f4c74876 with lease ID 0x3bf1f535b6ceea61: from storage DS-3e6a63ea-e623-4088-8abc-9563543f04e9 node DatanodeRegistration(127.0.0.1:42009, datanodeUuid=65fdf03d-c3df-49c9-aab8-3a2af9efceef, infoPort=37901, infoSecurePort=0, ipcPort=44373, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:16:40,925 WARN [Thread-140 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data6/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,925 WARN [Thread-139 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data5/current/BP-700201894-172.17.0.2-1733343399373/current, will proceed with Du for space computation calculation, 2024-12-04T20:16:40,952 WARN [Thread-129 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-04T20:16:40,957 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe71bf85ee8c11c81 with lease ID 0x3bf1f535b6ceea62: Processing first storage report for DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351 from datanode DatanodeRegistration(127.0.0.1:37737, datanodeUuid=28e2ccbb-c2af-4818-8139-7a6bd3c977ec, infoPort=44367, infoSecurePort=0, ipcPort=45181, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe71bf85ee8c11c81 with lease ID 0x3bf1f535b6ceea62: from storage DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351 node DatanodeRegistration(127.0.0.1:37737, datanodeUuid=28e2ccbb-c2af-4818-8139-7a6bd3c977ec, infoPort=44367, infoSecurePort=0, ipcPort=45181, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:16:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe71bf85ee8c11c81 with lease ID 0x3bf1f535b6ceea62: Processing first storage report for DS-774ca99f-2ea6-4141-86b5-1f949f46845e from datanode DatanodeRegistration(127.0.0.1:37737, datanodeUuid=28e2ccbb-c2af-4818-8139-7a6bd3c977ec, infoPort=44367, infoSecurePort=0, ipcPort=45181, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373) 2024-12-04T20:16:40,958 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe71bf85ee8c11c81 with lease ID 0x3bf1f535b6ceea62: from storage DS-774ca99f-2ea6-4141-86b5-1f949f46845e node DatanodeRegistration(127.0.0.1:37737, datanodeUuid=28e2ccbb-c2af-4818-8139-7a6bd3c977ec, infoPort=44367, infoSecurePort=0, ipcPort=45181, storageInfo=lv=-57;cid=testClusterID;nsid=751704312;c=1733343399373), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-04T20:16:41,127 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2 2024-12-04T20:16:41,199 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/zookeeper_0, clientPort=55668, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-04T20:16:41,209 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=55668 2024-12-04T20:16:41,223 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:41,226 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:41,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:16:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:16:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741825_1001 (size=7) 2024-12-04T20:16:41,873 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 with version=8 2024-12-04T20:16:41,874 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/hbase-staging 2024-12-04T20:16:42,111 INFO [Time-limited test {}] client.ConnectionUtils(128): master/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:16:42,119 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,120 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,124 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:16:42,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,124 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:16:42,247 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-04T20:16:42,305 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-04T20:16:42,314 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-04T20:16:42,318 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:16:42,346 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 58309 (auto-detected) 2024-12-04T20:16:42,347 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-04T20:16:42,365 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:37947 2024-12-04T20:16:42,383 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37947 connecting to ZooKeeper ensemble=127.0.0.1:55668 2024-12-04T20:16:42,409 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:379470x0, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:16:42,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37947-0x100a6e0b14d0000 connected 2024-12-04T20:16:42,437 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,440 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,449 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:16:42,452 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061, hbase.cluster.distributed=false 2024-12-04T20:16:42,473 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:16:42,476 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37947 2024-12-04T20:16:42,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37947 2024-12-04T20:16:42,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37947 2024-12-04T20:16:42,477 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37947 2024-12-04T20:16:42,478 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37947 2024-12-04T20:16:42,563 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:16:42,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,565 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:16:42,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,565 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:16:42,568 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:16:42,569 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:16:42,570 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45313 2024-12-04T20:16:42,572 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45313 connecting to ZooKeeper ensemble=127.0.0.1:55668 2024-12-04T20:16:42,573 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,575 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,581 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:453130x0, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:16:42,581 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:453130x0, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:16:42,581 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45313-0x100a6e0b14d0001 connected 2024-12-04T20:16:42,585 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:16:42,592 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:16:42,594 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:16:42,600 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:16:42,600 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-04T20:16:42,601 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45313 2024-12-04T20:16:42,604 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45313 2024-12-04T20:16:42,605 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-04T20:16:42,608 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45313 2024-12-04T20:16:42,624 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:16:42,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,625 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:16:42,625 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,626 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:16:42,626 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:16:42,626 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:16:42,627 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:39001 2024-12-04T20:16:42,628 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:39001 connecting to ZooKeeper ensemble=127.0.0.1:55668 2024-12-04T20:16:42,629 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,632 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,636 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:390010x0, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:16:42,637 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:39001-0x100a6e0b14d0002 connected 2024-12-04T20:16:42,637 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:16:42,638 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:16:42,638 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:16:42,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:16:42,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:16:42,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=39001 2024-12-04T20:16:42,642 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=39001 2024-12-04T20:16:42,645 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=39001 2024-12-04T20:16:42,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=39001 2024-12-04T20:16:42,646 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=39001 2024-12-04T20:16:42,661 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/c2ef38372881:0 server-side Connection retries=45 2024-12-04T20:16:42,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,661 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-04T20:16:42,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-04T20:16:42,662 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-04T20:16:42,662 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-04T20:16:42,662 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-04T20:16:42,663 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40377 2024-12-04T20:16:42,664 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40377 connecting to ZooKeeper ensemble=127.0.0.1:55668 2024-12-04T20:16:42,665 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,667 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,672 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:403770x0, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-04T20:16:42,672 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:403770x0, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:16:42,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40377-0x100a6e0b14d0003 connected 2024-12-04T20:16:42,673 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-04T20:16:42,674 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-04T20:16:42,675 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-04T20:16:42,676 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-04T20:16:42,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40377 2024-12-04T20:16:42,677 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40377 2024-12-04T20:16:42,678 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40377 2024-12-04T20:16:42,681 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40377 2024-12-04T20:16:42,684 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40377 2024-12-04T20:16:42,696 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;c2ef38372881:37947 2024-12-04T20:16:42,697 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/c2ef38372881,37947,1733343401962 2024-12-04T20:16:42,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,706 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/c2ef38372881,37947,1733343401962 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,727 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-04T20:16:42,728 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/c2ef38372881,37947,1733343401962 from backup master directory 2024-12-04T20:16:42,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/c2ef38372881,37947,1733343401962 2024-12-04T20:16:42,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-04T20:16:42,731 WARN [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:16:42,731 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=c2ef38372881,37947,1733343401962 2024-12-04T20:16:42,733 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-04T20:16:42,735 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-04T20:16:42,786 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/hbase.id] with ID: 6acde3c5-475c-4c4e-93e8-348877078bfd 2024-12-04T20:16:42,786 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/.tmp/hbase.id 2024-12-04T20:16:42,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:16:42,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:16:42,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741826_1002 (size=42) 2024-12-04T20:16:42,801 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/.tmp/hbase.id]:[hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/hbase.id] 2024-12-04T20:16:42,845 INFO [master/c2ef38372881:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-04T20:16:42,848 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-04T20:16:42,865 INFO [master/c2ef38372881:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 14ms. 2024-12-04T20:16:42,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,868 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:42,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:16:42,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:16:42,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741827_1003 (size=196) 2024-12-04T20:16:42,901 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:16:42,902 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-04T20:16:42,907 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:16:42,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:16:42,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741828_1004 (size=1189) 2024-12-04T20:16:42,953 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store 2024-12-04T20:16:42,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:16:42,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:16:42,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741829_1005 (size=34) 2024-12-04T20:16:42,981 INFO [master/c2ef38372881:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-04T20:16:42,984 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:42,985 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:16:42,985 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:16:42,986 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:16:42,987 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:16:42,987 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:16:42,987 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:16:42,988 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343402985Disabling compacts and flushes for region at 1733343402985Disabling writes for close at 1733343402987 (+2 ms)Writing region close event to WAL at 1733343402987Closed at 1733343402987 2024-12-04T20:16:42,990 WARN [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/.initializing 2024-12-04T20:16:42,991 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/WALs/c2ef38372881,37947,1733343401962 2024-12-04T20:16:42,998 INFO [master/c2ef38372881:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:43,013 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C37947%2C1733343401962, suffix=, logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/WALs/c2ef38372881,37947,1733343401962, archiveDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/oldWALs, maxLogs=10 2024-12-04T20:16:43,046 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/WALs/c2ef38372881,37947,1733343401962/c2ef38372881%2C37947%2C1733343401962.1733343403020, exclude list is [], retry=0 2024-12-04T20:16:43,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:43,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:43,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:43,067 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:43,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-04T20:16:43,102 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-04T20:16:43,157 INFO [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/WALs/c2ef38372881,37947,1733343401962/c2ef38372881%2C37947%2C1733343401962.1733343403020 2024-12-04T20:16:43,158 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:43,158 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:43,158 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:43,161 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,162 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,192 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,213 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-04T20:16:43,216 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,218 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:43,219 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,222 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-04T20:16:43,222 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:43,223 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,225 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-04T20:16:43,225 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:43,226 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,228 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-04T20:16:43,229 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,229 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:43,230 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,233 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,234 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,239 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,239 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,242 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:16:43,246 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-04T20:16:43,250 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:43,251 INFO [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66662739, jitterRate=-0.006647780537605286}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:16:43,256 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733343403172Initializing all the Stores at 1733343403174 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343403174Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343403175 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343403175Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343403175Cleaning up temporary data from old regions at 1733343403240 (+65 ms)Region opened successfully at 1733343403256 (+16 ms) 2024-12-04T20:16:43,258 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-04T20:16:43,286 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ca258d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:16:43,311 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-04T20:16:43,320 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-04T20:16:43,321 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-04T20:16:43,324 INFO [master/c2ef38372881:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-04T20:16:43,325 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-04T20:16:43,330 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-12-04T20:16:43,330 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-04T20:16:43,357 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-04T20:16:43,366 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-04T20:16:43,368 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-04T20:16:43,370 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-04T20:16:43,371 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-04T20:16:43,372 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-04T20:16:43,374 INFO [master/c2ef38372881:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-04T20:16:43,378 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-04T20:16:43,379 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-04T20:16:43,380 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-04T20:16:43,381 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-04T20:16:43,396 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-04T20:16:43,397 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,400 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,403 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=c2ef38372881,37947,1733343401962, sessionid=0x100a6e0b14d0000, setting cluster-up flag (Was=false) 2024-12-04T20:16:43,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,412 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,416 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-04T20:16:43,418 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,37947,1733343401962 2024-12-04T20:16:43,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,422 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:43,426 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-04T20:16:43,427 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=c2ef38372881,37947,1733343401962 2024-12-04T20:16:43,434 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-04T20:16:43,487 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(746): ClusterId : 6acde3c5-475c-4c4e-93e8-348877078bfd 2024-12-04T20:16:43,487 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(746): ClusterId : 6acde3c5-475c-4c4e-93e8-348877078bfd 2024-12-04T20:16:43,487 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(746): ClusterId : 6acde3c5-475c-4c4e-93e8-348877078bfd 2024-12-04T20:16:43,490 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:16:43,490 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:16:43,490 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-04T20:16:43,494 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:16:43,494 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:16:43,494 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-04T20:16:43,495 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:16:43,495 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:16:43,495 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-04T20:16:43,498 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:16:43,498 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:16:43,498 DEBUG [RS:2;c2ef38372881:40377 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f26ac1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:16:43,498 DEBUG [RS:1;c2ef38372881:39001 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6aaf2b62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:16:43,499 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-04T20:16:43,500 DEBUG [RS:0;c2ef38372881:45313 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7fb53ddf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=c2ef38372881/172.17.0.2:0 2024-12-04T20:16:43,504 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-04T20:16:43,511 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;c2ef38372881:45313 2024-12-04T20:16:43,514 INFO [RS:0;c2ef38372881:45313 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:16:43,514 INFO [RS:0;c2ef38372881:45313 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:16:43,514 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:16:43,516 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;c2ef38372881:40377 2024-12-04T20:16:43,516 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;c2ef38372881:39001 2024-12-04T20:16:43,516 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-04T20:16:43,517 INFO [RS:1;c2ef38372881:39001 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:16:43,517 INFO [RS:2;c2ef38372881:40377 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-04T20:16:43,517 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,37947,1733343401962 with port=45313, startcode=1733343402534 2024-12-04T20:16:43,517 INFO [RS:1;c2ef38372881:39001 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:16:43,517 INFO [RS:2;c2ef38372881:40377 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-04T20:16:43,517 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:16:43,517 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-04T20:16:43,518 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,37947,1733343401962 with port=39001, startcode=1733343402624 2024-12-04T20:16:43,518 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,37947,1733343401962 with port=40377, startcode=1733343402660 2024-12-04T20:16:43,525 INFO [master/c2ef38372881:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-04T20:16:43,529 DEBUG [RS:0;c2ef38372881:45313 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:16:43,529 DEBUG [RS:2;c2ef38372881:40377 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:16:43,529 DEBUG [RS:1;c2ef38372881:39001 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-04T20:16:43,532 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: c2ef38372881,37947,1733343401962 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-04T20:16:43,541 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:16:43,541 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:16:43,541 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:16:43,541 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/c2ef38372881:0, corePoolSize=5, maxPoolSize=5 2024-12-04T20:16:43,542 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/c2ef38372881:0, corePoolSize=10, maxPoolSize=10 2024-12-04T20:16:43,542 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,542 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:16:43,542 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,549 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733343433549 2024-12-04T20:16:43,551 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-04T20:16:43,552 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-04T20:16:43,556 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-04T20:16:43,557 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-04T20:16:43,557 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-04T20:16:43,557 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-04T20:16:43,560 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:16:43,561 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-04T20:16:43,560 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,565 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-04T20:16:43,567 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-04T20:16:43,567 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49471, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:16:43,567 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60569, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:16:43,567 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43159, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-04T20:16:43,568 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-04T20:16:43,569 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-04T20:16:43,569 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,570 INFO [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-04T20:16:43,570 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-04T20:16:43,571 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343403571,5,FailOnTimeoutGroup] 2024-12-04T20:16:43,574 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-04T20:16:43,577 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343403571,5,FailOnTimeoutGroup] 2024-12-04T20:16:43,577 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,577 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-04T20:16:43,579 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,579 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,582 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,584 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:16:43,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:16:43,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741831_1007 (size=1321) 2024-12-04T20:16:43,618 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,618 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-04T20:16:43,618 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,619 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 2024-12-04T20:16:43,619 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44433 2024-12-04T20:16:43,619 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:16:43,619 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 2024-12-04T20:16:43,623 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-12-04T20:16:43,623 WARN [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-12-04T20:16:43,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:16:43,624 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 2024-12-04T20:16:43,624 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44433 2024-12-04T20:16:43,624 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:16:43,624 DEBUG [RS:0;c2ef38372881:45313 {}] zookeeper.ZKUtil(111): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,625 WARN [RS:0;c2ef38372881:45313 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:16:43,625 INFO [RS:0;c2ef38372881:45313 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:43,625 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,627 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,45313,1733343402534] 2024-12-04T20:16:43,627 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:16:43,628 DEBUG [RS:1;c2ef38372881:39001 {}] zookeeper.ZKUtil(111): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,628 WARN [RS:1;c2ef38372881:39001 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:16:43,628 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,39001,1733343402624] 2024-12-04T20:16:43,628 INFO [RS:1;c2ef38372881:39001 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:43,628 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:16:43,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:16:43,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741832_1008 (size=32) 2024-12-04T20:16:43,645 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:43,647 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:16:43,650 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:16:43,650 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:43,651 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:16:43,653 INFO [RS:0;c2ef38372881:45313 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:16:43,653 INFO [RS:1;c2ef38372881:39001 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:16:43,654 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:16:43,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:43,656 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:16:43,659 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:16:43,659 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:43,660 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:16:43,664 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:16:43,664 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:43,665 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:43,665 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:16:43,667 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740 2024-12-04T20:16:43,668 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740 2024-12-04T20:16:43,668 INFO [RS:1;c2ef38372881:39001 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:16:43,668 INFO [RS:0;c2ef38372881:45313 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:16:43,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:16:43,671 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:16:43,671 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:16:43,674 INFO [RS:0;c2ef38372881:45313 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:16:43,674 INFO [RS:1;c2ef38372881:39001 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:16:43,674 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,674 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,674 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:16:43,675 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:16:43,675 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:16:43,678 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:43,679 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72486896, jitterRate=0.0801389217376709}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:16:43,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733343403645Initializing all the Stores at 1733343403647 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343403647Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343403647Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343403647Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343403647Cleaning up temporary data from old regions at 1733343403671 (+24 ms)Region opened successfully at 1733343403680 (+9 ms) 2024-12-04T20:16:43,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:16:43,680 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:16:43,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:16:43,680 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:16:43,681 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:16:43,681 INFO [RS:0;c2ef38372881:45313 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:16:43,681 INFO [RS:1;c2ef38372881:39001 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:16:43,682 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:16:43,682 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343403680Disabling compacts and flushes for region at 1733343403680Disabling writes for close at 1733343403681 (+1 ms)Writing region close event to WAL at 1733343403682 (+1 ms)Closed at 1733343403682 2024-12-04T20:16:43,683 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,683 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,683 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,683 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:16:43,684 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,684 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,685 DEBUG [RS:0;c2ef38372881:45313 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,685 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,686 DEBUG [RS:1;c2ef38372881:39001 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,686 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:16:43,686 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-04T20:16:43,686 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,45313,1733343402534-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,687 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,688 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,688 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,39001,1733343402624-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:16:43,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-04T20:16:43,703 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:16:43,706 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:16:43,706 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:16:43,707 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-04T20:16:43,708 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,39001,1733343402624-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,708 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,45313,1733343402534-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,709 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,709 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,709 INFO [RS:1;c2ef38372881:39001 {}] regionserver.Replication(171): c2ef38372881,39001,1733343402624 started 2024-12-04T20:16:43,709 INFO [RS:0;c2ef38372881:45313 {}] regionserver.Replication(171): c2ef38372881,45313,1733343402534 started 2024-12-04T20:16:43,724 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(2659): reportForDuty to master=c2ef38372881,37947,1733343401962 with port=40377, startcode=1733343402660 2024-12-04T20:16:43,726 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(363): Checking decommissioned status of RegionServer c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,726 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37947 {}] master.ServerManager(517): Registering regionserver=c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,727 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,727 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,728 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,39001,1733343402624, RpcServer on c2ef38372881/172.17.0.2:39001, sessionid=0x100a6e0b14d0002 2024-12-04T20:16:43,728 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,45313,1733343402534, RpcServer on c2ef38372881/172.17.0.2:45313, sessionid=0x100a6e0b14d0001 2024-12-04T20:16:43,729 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:16:43,729 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:16:43,729 DEBUG [RS:1;c2ef38372881:39001 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,729 DEBUG [RS:0;c2ef38372881:45313 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,729 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,45313,1733343402534' 2024-12-04T20:16:43,729 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,39001,1733343402624' 2024-12-04T20:16:43,729 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:16:43,729 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:16:43,731 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:16:43,731 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:16:43,731 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 2024-12-04T20:16:43,731 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:44433 2024-12-04T20:16:43,731 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-04T20:16:43,731 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:16:43,731 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:16:43,732 DEBUG [RS:0;c2ef38372881:45313 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,45313,1733343402534 2024-12-04T20:16:43,732 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,45313,1733343402534' 2024-12-04T20:16:43,732 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:16:43,732 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:16:43,732 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:16:43,732 DEBUG [RS:1;c2ef38372881:39001 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,39001,1733343402624 2024-12-04T20:16:43,732 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,39001,1733343402624' 2024-12-04T20:16:43,732 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:16:43,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:16:43,733 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:16:43,733 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:16:43,733 DEBUG [RS:1;c2ef38372881:39001 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:16:43,733 INFO [RS:1;c2ef38372881:39001 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:16:43,733 INFO [RS:1;c2ef38372881:39001 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:16:43,734 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [c2ef38372881,40377,1733343402660] 2024-12-04T20:16:43,734 DEBUG [RS:2;c2ef38372881:40377 {}] zookeeper.ZKUtil(111): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,734 DEBUG [RS:0;c2ef38372881:45313 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:16:43,734 WARN [RS:2;c2ef38372881:40377 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-04T20:16:43,734 INFO [RS:0;c2ef38372881:45313 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:16:43,735 INFO [RS:0;c2ef38372881:45313 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:16:43,735 INFO [RS:2;c2ef38372881:40377 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:43,735 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,740 INFO [RS:2;c2ef38372881:40377 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-04T20:16:43,743 INFO [RS:2;c2ef38372881:40377 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-04T20:16:43,744 INFO [RS:2;c2ef38372881:40377 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:16:43,744 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,745 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-04T20:16:43,747 INFO [RS:2;c2ef38372881:40377 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-04T20:16:43,747 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,747 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,748 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,748 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,748 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,748 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,748 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0, corePoolSize=2, maxPoolSize=2 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,749 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/c2ef38372881:0, corePoolSize=1, maxPoolSize=1 2024-12-04T20:16:43,750 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,750 DEBUG [RS:2;c2ef38372881:40377 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/c2ef38372881:0, corePoolSize=3, maxPoolSize=3 2024-12-04T20:16:43,750 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,750 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,751 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,751 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,751 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,751 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,40377,1733343402660-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:16:43,768 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-04T20:16:43,768 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,40377,1733343402660-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,768 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,769 INFO [RS:2;c2ef38372881:40377 {}] regionserver.Replication(171): c2ef38372881,40377,1733343402660 started 2024-12-04T20:16:43,781 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:43,782 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1482): Serving as c2ef38372881,40377,1733343402660, RpcServer on c2ef38372881/172.17.0.2:40377, sessionid=0x100a6e0b14d0003 2024-12-04T20:16:43,782 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-04T20:16:43,782 DEBUG [RS:2;c2ef38372881:40377 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,782 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,40377,1733343402660' 2024-12-04T20:16:43,782 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-04T20:16:43,783 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager c2ef38372881,40377,1733343402660 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'c2ef38372881,40377,1733343402660' 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-04T20:16:43,784 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-04T20:16:43,785 DEBUG [RS:2;c2ef38372881:40377 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-04T20:16:43,785 INFO [RS:2;c2ef38372881:40377 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-04T20:16:43,785 INFO [RS:2;c2ef38372881:40377 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-04T20:16:43,839 INFO [RS:0;c2ef38372881:45313 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:43,839 INFO [RS:1;c2ef38372881:39001 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:43,843 INFO [RS:0;c2ef38372881:45313 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C45313%2C1733343402534, suffix=, logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,45313,1733343402534, archiveDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs, maxLogs=32 2024-12-04T20:16:43,843 INFO [RS:1;c2ef38372881:39001 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C39001%2C1733343402624, suffix=, logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624, archiveDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs, maxLogs=32 2024-12-04T20:16:43,857 WARN [c2ef38372881:37947 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-04T20:16:43,858 DEBUG [RS:0;c2ef38372881:45313 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,45313,1733343402534/c2ef38372881%2C45313%2C1733343402534.1733343403847, exclude list is [], retry=0 2024-12-04T20:16:43,858 DEBUG [RS:1;c2ef38372881:39001 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624/c2ef38372881%2C39001%2C1733343402624.1733343403847, exclude list is [], retry=0 2024-12-04T20:16:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:43,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:43,864 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:43,886 INFO [RS:2;c2ef38372881:40377 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:43,889 INFO [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C40377%2C1733343402660, suffix=, logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660, archiveDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs, maxLogs=32 2024-12-04T20:16:43,891 INFO [RS:0;c2ef38372881:45313 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,45313,1733343402534/c2ef38372881%2C45313%2C1733343402534.1733343403847 2024-12-04T20:16:43,893 DEBUG [RS:0;c2ef38372881:45313 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:43,897 INFO [RS:1;c2ef38372881:39001 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624/c2ef38372881%2C39001%2C1733343402624.1733343403847 2024-12-04T20:16:43,897 DEBUG [RS:1;c2ef38372881:39001 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:16:43,904 DEBUG [RS:2;c2ef38372881:40377 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660/c2ef38372881%2C40377%2C1733343402660.1733343403891, exclude list is [], retry=0 2024-12-04T20:16:43,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:43,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:43,909 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:43,936 INFO [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660/c2ef38372881%2C40377%2C1733343402660.1733343403891 2024-12-04T20:16:43,936 DEBUG [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:44,113 DEBUG [c2ef38372881:37947 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-04T20:16:44,123 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(204): Hosts are {c2ef38372881=0} racks are {/default-rack=0} 2024-12-04T20:16:44,129 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T20:16:44,129 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T20:16:44,130 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T20:16:44,130 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T20:16:44,130 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T20:16:44,130 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T20:16:44,130 INFO [c2ef38372881:37947 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T20:16:44,130 INFO [c2ef38372881:37947 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T20:16:44,130 INFO [c2ef38372881:37947 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T20:16:44,130 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T20:16:44,136 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=c2ef38372881,40377,1733343402660 2024-12-04T20:16:44,140 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,40377,1733343402660, state=OPENING 2024-12-04T20:16:44,145 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-04T20:16:44,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:44,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:44,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:44,146 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:16:44,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,146 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,147 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,147 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,148 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-04T20:16:44,150 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,40377,1733343402660}] 2024-12-04T20:16:44,321 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:16:44,323 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36727, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:16:44,333 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-04T20:16:44,334 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:44,335 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-04T20:16:44,338 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=c2ef38372881%2C40377%2C1733343402660.meta, suffix=.meta, logDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660, archiveDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs, maxLogs=32 2024-12-04T20:16:44,353 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660/c2ef38372881%2C40377%2C1733343402660.meta.1733343404339.meta, exclude list is [], retry=0 2024-12-04T20:16:44,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:44,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:44,357 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:44,385 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,40377,1733343402660/c2ef38372881%2C40377%2C1733343402660.meta.1733343404339.meta 2024-12-04T20:16:44,385 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:44,385 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:44,387 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-04T20:16:44,389 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-04T20:16:44,394 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-04T20:16:44,397 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-04T20:16:44,398 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:44,398 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-04T20:16:44,398 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-04T20:16:44,401 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-04T20:16:44,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-04T20:16:44,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:44,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:44,404 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-04T20:16:44,405 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-04T20:16:44,406 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:44,406 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:44,407 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-04T20:16:44,408 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-04T20:16:44,408 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:44,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:44,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-04T20:16:44,411 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-04T20:16:44,411 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:44,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-04T20:16:44,412 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-04T20:16:44,414 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740 2024-12-04T20:16:44,416 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740 2024-12-04T20:16:44,419 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-04T20:16:44,419 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-04T20:16:44,420 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-04T20:16:44,422 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-04T20:16:44,424 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68162369, jitterRate=0.015698447823524475}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-04T20:16:44,424 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-04T20:16:44,425 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733343404399Writing region info on filesystem at 1733343404399Initializing all the Stores at 1733343404401 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343404401Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343404401Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343404401Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733343404401Cleaning up temporary data from old regions at 1733343404419 (+18 ms)Running coprocessor post-open hooks at 1733343404424 (+5 ms)Region opened successfully at 1733343404425 (+1 ms) 2024-12-04T20:16:44,432 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733343404314 2024-12-04T20:16:44,443 DEBUG [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-04T20:16:44,444 INFO [RS_OPEN_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-04T20:16:44,445 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,40377,1733343402660 2024-12-04T20:16:44,447 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as c2ef38372881,40377,1733343402660, state=OPEN 2024-12-04T20:16:44,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:16:44,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:16:44,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:16:44,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-04T20:16:44,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,449 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-04T20:16:44,449 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=c2ef38372881,40377,1733343402660 2024-12-04T20:16:44,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-04T20:16:44,455 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=c2ef38372881,40377,1733343402660 in 300 msec 2024-12-04T20:16:44,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-04T20:16:44,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 763 msec 2024-12-04T20:16:44,464 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-04T20:16:44,464 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-04T20:16:44,482 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:16:44,484 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,40377,1733343402660, seqNum=-1] 2024-12-04T20:16:44,508 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:16:44,510 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57713, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:16:44,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.0660 sec 2024-12-04T20:16:44,530 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733343404530, completionTime=-1 2024-12-04T20:16:44,532 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-04T20:16:44,532 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-04T20:16:44,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-04T20:16:44,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733343464555 2024-12-04T20:16:44,555 INFO [master/c2ef38372881:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733343524555 2024-12-04T20:16:44,556 INFO [master/c2ef38372881:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-12-04T20:16:44,557 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-04T20:16:44,581 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,582 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,582 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,584 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-c2ef38372881:37947, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,584 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,585 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,592 DEBUG [master/c2ef38372881:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-04T20:16:44,616 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 1.884sec 2024-12-04T20:16:44,617 INFO [master/c2ef38372881:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-04T20:16:44,618 INFO [master/c2ef38372881:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-04T20:16:44,619 INFO [master/c2ef38372881:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-04T20:16:44,620 INFO [master/c2ef38372881:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-04T20:16:44,620 INFO [master/c2ef38372881:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-04T20:16:44,620 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-04T20:16:44,621 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-04T20:16:44,625 DEBUG [master/c2ef38372881:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-04T20:16:44,626 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-04T20:16:44,626 INFO [master/c2ef38372881:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=c2ef38372881,37947,1733343401962-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:44,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@287698dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:16:44,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request c2ef38372881,37947,-1 for getting cluster id 2024-12-04T20:16:44,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-04T20:16:44,713 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '6acde3c5-475c-4c4e-93e8-348877078bfd' 2024-12-04T20:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-04T20:16:44,715 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "6acde3c5-475c-4c4e-93e8-348877078bfd" 2024-12-04T20:16:44,718 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21685e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:16:44,718 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [c2ef38372881,37947,-1] 2024-12-04T20:16:44,720 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-04T20:16:44,722 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:16:44,723 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-04T20:16:44,726 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fd88229, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-04T20:16:44,727 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-04T20:16:44,735 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=c2ef38372881,40377,1733343402660, seqNum=-1] 2024-12-04T20:16:44,736 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:16:44,738 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:37014, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:16:44,756 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=c2ef38372881,37947,1733343401962 2024-12-04T20:16:44,756 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:44433/hbase 2024-12-04T20:16:44,769 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=362, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=5411 2024-12-04T20:16:44,785 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:44,789 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:44,790 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:44,795 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-41813725, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-41813725, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:44,809 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-41813725/hregion-41813725.1733343404797, exclude list is [], retry=0 2024-12-04T20:16:44,814 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:44,814 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:44,814 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:44,842 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-41813725/hregion-41813725.1733343404797 2024-12-04T20:16:44,842 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:44,842 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 47a7f8aba972a07fa4a7adbf89b51a7d, NAME => 'testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:16:44,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741838_1014 (size=64) 2024-12-04T20:16:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741838_1014 (size=64) 2024-12-04T20:16:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741838_1014 (size=64) 2024-12-04T20:16:44,856 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:44,858 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,860 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a7f8aba972a07fa4a7adbf89b51a7d columnFamilyName a 2024-12-04T20:16:44,860 DEBUG [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:44,861 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] regionserver.HStore(327): Store=47a7f8aba972a07fa4a7adbf89b51a7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:44,861 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,863 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,864 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,864 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,867 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:44,871 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:44,871 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47a7f8aba972a07fa4a7adbf89b51a7d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59782319, jitterRate=-0.10917402803897858}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:16:44,872 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47a7f8aba972a07fa4a7adbf89b51a7d: Writing region info on filesystem at 1733343404856Initializing all the Stores at 1733343404857 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343404857Cleaning up temporary data from old regions at 1733343404864 (+7 ms)Region opened successfully at 1733343404872 (+8 ms) 2024-12-04T20:16:44,872 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47a7f8aba972a07fa4a7adbf89b51a7d, disabling compactions & flushes 2024-12-04T20:16:44,873 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. 2024-12-04T20:16:44,873 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. 2024-12-04T20:16:44,873 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. after waiting 0 ms 2024-12-04T20:16:44,873 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. 2024-12-04T20:16:44,873 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. 2024-12-04T20:16:44,873 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47a7f8aba972a07fa4a7adbf89b51a7d: Waiting for close lock at 1733343404872Disabling compacts and flushes for region at 1733343404872Disabling writes for close at 1733343404873 (+1 ms)Writing region close event to WAL at 1733343404873Closed at 1733343404873 2024-12-04T20:16:44,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741837_1013 (size=175) 2024-12-04T20:16:44,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741837_1013 (size=175) 2024-12-04T20:16:44,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741837_1013 (size=175) 2024-12-04T20:16:44,884 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:44,884 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-41813725:(num 1733343404797) 2024-12-04T20:16:44,886 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T20:16:44,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741839_1015 (size=469) 2024-12-04T20:16:44,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741839_1015 (size=469) 2024-12-04T20:16:44,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741839_1015 (size=469) 2024-12-04T20:16:44,924 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T20:16:44,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741840_1016 (size=322) 2024-12-04T20:16:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741840_1016 (size=322) 2024-12-04T20:16:44,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741840_1016 (size=322) 2024-12-04T20:16:44,979 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1, size=469 (469bytes) 2024-12-04T20:16:44,980 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-04T20:16:44,980 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-04T20:16:44,980 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1 2024-12-04T20:16:44,985 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1 after 3ms 2024-12-04T20:16:45,011 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,012 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1 took 34ms 2024-12-04T20:16:45,016 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1 so closing down 2024-12-04T20:16:45,017 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:45,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T20:16:45,043 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T20:16:45,044 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:45,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741841_1017 (size=469) 2024-12-04T20:16:45,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741841_1017 (size=469) 2024-12-04T20:16:45,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741841_1017 (size=469) 2024-12-04T20:16:45,055 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:45,058 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002 2024-12-04T20:16:45,063 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-04T20:16:45,063 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1, journal: Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1, size=469 (469bytes) at 1733343404979Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1 so closing down at 1733343405017 (+38 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp at 1733343405043 (+26 ms)3 split writer threads finished at 1733343405044 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733343405055 (+11 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002 at 1733343405058 (+3 ms)Processed 2 edits across 1 Regions in 45 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733343405063 (+5 ms) 2024-12-04T20:16:45,075 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2, size=322 (322bytes) 2024-12-04T20:16:45,075 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2 2024-12-04T20:16:45,076 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2 after 1ms 2024-12-04T20:16:45,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,101 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2 took 26ms 2024-12-04T20:16:45,104 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2 so closing down 2024-12-04T20:16:45,104 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:45,106 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T20:16:45,128 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T20:16:45,129 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741842_1018 (size=322) 2024-12-04T20:16:45,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741842_1018 (size=322) 2024-12-04T20:16:45,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741842_1018 (size=322) 2024-12-04T20:16:45,138 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:45,163 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,165 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-04T20:16:45,167 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-04T20:16:45,167 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2, journal: Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2, size=322 (322bytes) at 1733343405075Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2 so closing down at 1733343405104 (+29 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002-wal-2.temp at 1733343405128 (+24 ms)3 split writer threads finished at 1733343405129 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733343405138 (+9 ms)Processed 1 edits across 1 Regions in 65 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733343405167 (+29 ms) 2024-12-04T20:16:45,167 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:45,169 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:45,180 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal.1733343405170, exclude list is [], retry=0 2024-12-04T20:16:45,183 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:45,184 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:45,184 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:45,208 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal.1733343405170 2024-12-04T20:16:45,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:45,208 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47a7f8aba972a07fa4a7adbf89b51a7d, NAME => 'testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:45,209 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:45,209 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,209 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,211 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,212 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47a7f8aba972a07fa4a7adbf89b51a7d columnFamilyName a 2024-12-04T20:16:45,212 DEBUG [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,213 INFO [StoreOpener-47a7f8aba972a07fa4a7adbf89b51a7d-1 {}] regionserver.HStore(327): Store=47a7f8aba972a07fa4a7adbf89b51a7d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,213 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,215 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,217 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,218 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002 2024-12-04T20:16:45,244 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,249 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002 2024-12-04T20:16:45,253 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47a7f8aba972a07fa4a7adbf89b51a7d 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T20:16:45,301 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/.tmp/a/d9596821710143a599b394473a4e06d7 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733343404884/Put/seqid=0 2024-12-04T20:16:45,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741844_1020 (size=5170) 2024-12-04T20:16:45,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741844_1020 (size=5170) 2024-12-04T20:16:45,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741844_1020 (size=5170) 2024-12-04T20:16:45,316 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/.tmp/a/d9596821710143a599b394473a4e06d7 2024-12-04T20:16:45,356 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/.tmp/a/d9596821710143a599b394473a4e06d7 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/a/d9596821710143a599b394473a4e06d7 2024-12-04T20:16:45,364 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/a/d9596821710143a599b394473a4e06d7, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T20:16:45,369 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 47a7f8aba972a07fa4a7adbf89b51a7d in 115ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T20:16:45,370 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/0000000000000000002 2024-12-04T20:16:45,371 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,371 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,374 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47a7f8aba972a07fa4a7adbf89b51a7d 2024-12-04T20:16:45,377 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/47a7f8aba972a07fa4a7adbf89b51a7d/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T20:16:45,378 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47a7f8aba972a07fa4a7adbf89b51a7d; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65203858, jitterRate=-0.028386801481246948}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:16:45,379 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47a7f8aba972a07fa4a7adbf89b51a7d: Writing region info on filesystem at 1733343405209Initializing all the Stores at 1733343405211 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405211Obtaining lock to block concurrent updates at 1733343405253 (+42 ms)Preparing flush snapshotting stores in 47a7f8aba972a07fa4a7adbf89b51a7d at 1733343405253Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733343405257 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733343404786.47a7f8aba972a07fa4a7adbf89b51a7d. at 1733343405257Flushing 47a7f8aba972a07fa4a7adbf89b51a7d/a: creating writer at 1733343405258 (+1 ms)Flushing 47a7f8aba972a07fa4a7adbf89b51a7d/a: appending metadata at 1733343405292 (+34 ms)Flushing 47a7f8aba972a07fa4a7adbf89b51a7d/a: closing flushed file at 1733343405294 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3fc93182: reopening flushed file at 1733343405354 (+60 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 47a7f8aba972a07fa4a7adbf89b51a7d in 115ms, sequenceid=2, compaction requested=false; wal=null at 1733343405369 (+15 ms)Cleaning up temporary data from old regions at 1733343405371 (+2 ms)Region opened successfully at 1733343405379 (+8 ms) 2024-12-04T20:16:45,404 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=373 (was 362) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:44433/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46106 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:45950 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36962 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36864 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:40898 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=5388 (was 5411) 2024-12-04T20:16:45,415 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=373, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=5386 2024-12-04T20:16:45,428 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:45,430 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:45,431 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:45,434 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-19411424, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-19411424, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:45,446 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-19411424/hregion-19411424.1733343405435, exclude list is [], retry=0 2024-12-04T20:16:45,450 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:45,450 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:45,450 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:45,474 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-19411424/hregion-19411424.1733343405435 2024-12-04T20:16:45,475 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:45,475 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f9caf57e887b4f2a5b0dec7ef5060141, NAME => 'testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:16:45,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741846_1022 (size=64) 2024-12-04T20:16:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741846_1022 (size=64) 2024-12-04T20:16:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741846_1022 (size=64) 2024-12-04T20:16:45,486 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:45,488 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,490 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9caf57e887b4f2a5b0dec7ef5060141 columnFamilyName a 2024-12-04T20:16:45,490 DEBUG [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,491 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] regionserver.HStore(327): Store=f9caf57e887b4f2a5b0dec7ef5060141/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,491 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,492 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,493 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,494 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,494 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,496 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,499 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:45,500 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f9caf57e887b4f2a5b0dec7ef5060141; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72946607, jitterRate=0.0869891494512558}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:16:45,500 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f9caf57e887b4f2a5b0dec7ef5060141: Writing region info on filesystem at 1733343405486Initializing all the Stores at 1733343405488 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405488Cleaning up temporary data from old regions at 1733343405494 (+6 ms)Region opened successfully at 1733343405500 (+6 ms) 2024-12-04T20:16:45,501 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f9caf57e887b4f2a5b0dec7ef5060141, disabling compactions & flushes 2024-12-04T20:16:45,501 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. 2024-12-04T20:16:45,501 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. 2024-12-04T20:16:45,501 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. after waiting 0 ms 2024-12-04T20:16:45,501 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. 2024-12-04T20:16:45,501 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. 2024-12-04T20:16:45,501 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f9caf57e887b4f2a5b0dec7ef5060141: Waiting for close lock at 1733343405500Disabling compacts and flushes for region at 1733343405500Disabling writes for close at 1733343405501 (+1 ms)Writing region close event to WAL at 1733343405501Closed at 1733343405501 2024-12-04T20:16:45,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741845_1021 (size=175) 2024-12-04T20:16:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741845_1021 (size=175) 2024-12-04T20:16:45,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741845_1021 (size=175) 2024-12-04T20:16:45,507 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:45,507 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-19411424:(num 1733343405435) 2024-12-04T20:16:45,508 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T20:16:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741847_1023 (size=469) 2024-12-04T20:16:45,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741847_1023 (size=469) 2024-12-04T20:16:45,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741847_1023 (size=469) 2024-12-04T20:16:45,542 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-04T20:16:45,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741848_1024 (size=322) 2024-12-04T20:16:45,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741848_1024 (size=322) 2024-12-04T20:16:45,575 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741848_1024 (size=322) 2024-12-04T20:16:45,589 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2, size=322 (322bytes) 2024-12-04T20:16:45,589 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2 2024-12-04T20:16:45,590 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2 after 1ms 2024-12-04T20:16:45,615 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,615 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2 took 26ms 2024-12-04T20:16:45,617 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2 so closing down 2024-12-04T20:16:45,617 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:45,619 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-04T20:16:45,642 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp 2024-12-04T20:16:45,642 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:45,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741849_1025 (size=322) 2024-12-04T20:16:45,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741849_1025 (size=322) 2024-12-04T20:16:45,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741849_1025 (size=322) 2024-12-04T20:16:45,650 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:45,652 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 2024-12-04T20:16:45,652 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-04T20:16:45,652 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2, journal: Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2, size=322 (322bytes) at 1733343405589Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2 so closing down at 1733343405617 (+28 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp at 1733343405642 (+25 ms)3 split writer threads finished at 1733343405642Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733343405650 (+8 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 at 1733343405652 (+2 ms)Processed 1 edits across 1 Regions in 37 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733343405652 2024-12-04T20:16:45,665 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1, size=469 (469bytes) 2024-12-04T20:16:45,665 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1 2024-12-04T20:16:45,666 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1 after 1ms 2024-12-04T20:16:45,692 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,693 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1 took 28ms 2024-12-04T20:16:45,695 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1 so closing down 2024-12-04T20:16:45,696 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:45,697 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-04T20:16:45,721 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp 2024-12-04T20:16:45,721 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:45,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741850_1026 (size=469) 2024-12-04T20:16:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741850_1026 (size=469) 2024-12-04T20:16:45,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741850_1026 (size=469) 2024-12-04T20:16:45,730 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:45,757 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,759 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002, length=322 2024-12-04T20:16:45,761 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 2024-12-04T20:16:45,761 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-04T20:16:45,762 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1, journal: Splitting hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1, size=469 (469bytes) at 1733343405665Finishing writing output for hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1 so closing down at 1733343405696 (+31 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp at 1733343405721 (+25 ms)3 split writer threads finished at 1733343405721Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733343405730 (+9 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 at 1733343405761 (+31 ms)Processed 2 edits across 1 Regions in 68 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733343405761 2024-12-04T20:16:45,762 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:45,764 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:45,775 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal.1733343405764, exclude list is [], retry=0 2024-12-04T20:16:45,778 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:45,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:45,779 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:45,802 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal.1733343405764 2024-12-04T20:16:45,803 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:45,803 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f9caf57e887b4f2a5b0dec7ef5060141, NAME => 'testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:45,803 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:45,803 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,803 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,805 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,807 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f9caf57e887b4f2a5b0dec7ef5060141 columnFamilyName a 2024-12-04T20:16:45,807 DEBUG [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,808 INFO [StoreOpener-f9caf57e887b4f2a5b0dec7ef5060141-1 {}] regionserver.HStore(327): Store=f9caf57e887b4f2a5b0dec7ef5060141/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,808 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,809 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,812 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 2024-12-04T20:16:45,837 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:45,839 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 2024-12-04T20:16:45,839 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f9caf57e887b4f2a5b0dec7ef5060141 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-04T20:16:45,852 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/.tmp/a/da70bcddf3224551b9a83c830802c43c is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733343405507/Put/seqid=0 2024-12-04T20:16:45,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741852_1028 (size=5170) 2024-12-04T20:16:45,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741852_1028 (size=5170) 2024-12-04T20:16:45,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741852_1028 (size=5170) 2024-12-04T20:16:45,862 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/.tmp/a/da70bcddf3224551b9a83c830802c43c 2024-12-04T20:16:45,871 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/.tmp/a/da70bcddf3224551b9a83c830802c43c as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/a/da70bcddf3224551b9a83c830802c43c 2024-12-04T20:16:45,879 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/a/da70bcddf3224551b9a83c830802c43c, entries=2, sequenceid=2, filesize=5.0 K 2024-12-04T20:16:45,879 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f9caf57e887b4f2a5b0dec7ef5060141 in 40ms, sequenceid=2, compaction requested=false; wal=null 2024-12-04T20:16:45,880 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/0000000000000000002 2024-12-04T20:16:45,881 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,881 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,884 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f9caf57e887b4f2a5b0dec7ef5060141 2024-12-04T20:16:45,887 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/f9caf57e887b4f2a5b0dec7ef5060141/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-04T20:16:45,888 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f9caf57e887b4f2a5b0dec7ef5060141; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63249934, jitterRate=-0.057502537965774536}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:16:45,889 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f9caf57e887b4f2a5b0dec7ef5060141: Writing region info on filesystem at 1733343405803Initializing all the Stores at 1733343405805 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405805Obtaining lock to block concurrent updates at 1733343405839 (+34 ms)Preparing flush snapshotting stores in f9caf57e887b4f2a5b0dec7ef5060141 at 1733343405839Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733343405839Flushing stores of testReplayEditsWrittenIntoWAL,,1733343405429.f9caf57e887b4f2a5b0dec7ef5060141. at 1733343405839Flushing f9caf57e887b4f2a5b0dec7ef5060141/a: creating writer at 1733343405839Flushing f9caf57e887b4f2a5b0dec7ef5060141/a: appending metadata at 1733343405851 (+12 ms)Flushing f9caf57e887b4f2a5b0dec7ef5060141/a: closing flushed file at 1733343405851Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2c625265: reopening flushed file at 1733343405869 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for f9caf57e887b4f2a5b0dec7ef5060141 in 40ms, sequenceid=2, compaction requested=false; wal=null at 1733343405879 (+10 ms)Cleaning up temporary data from old regions at 1733343405881 (+2 ms)Region opened successfully at 1733343405888 (+7 ms) 2024-12-04T20:16:45,905 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=383 (was 373) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:37038 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:45950 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46170 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:40724 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:40952 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36864 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=138 (was 138), ProcessCount=11 (was 11), AvailableMemoryMB=5371 (was 5386) 2024-12-04T20:16:45,915 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=383, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=138, ProcessCount=11, AvailableMemoryMB=5370 2024-12-04T20:16:45,929 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:45,931 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:45,932 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:45,935 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-65412602, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-65412602, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:45,947 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-65412602/hregion-65412602.1733343405936, exclude list is [], retry=0 2024-12-04T20:16:45,950 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:45,950 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:45,950 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:45,974 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-65412602/hregion-65412602.1733343405936 2024-12-04T20:16:45,975 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:45,975 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 49bf9bd65a1bae51a9870869671e6eee, NAME => 'testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:16:45,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741854_1030 (size=64) 2024-12-04T20:16:45,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741854_1030 (size=64) 2024-12-04T20:16:45,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741854_1030 (size=64) 2024-12-04T20:16:45,989 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:45,990 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:45,992 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName a 2024-12-04T20:16:45,992 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,993 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,993 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:45,995 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName b 2024-12-04T20:16:45,995 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,995 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,996 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:45,997 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName c 2024-12-04T20:16:45,997 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:45,998 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:45,998 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:45,999 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:46,000 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:46,001 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:46,001 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:46,002 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:46,004 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:46,006 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:46,007 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49bf9bd65a1bae51a9870869671e6eee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64476244, jitterRate=-0.03922909498214722}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:46,007 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49bf9bd65a1bae51a9870869671e6eee: Writing region info on filesystem at 1733343405989Initializing all the Stores at 1733343405990 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405990Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405990Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343405990Cleaning up temporary data from old regions at 1733343406001 (+11 ms)Region opened successfully at 1733343406007 (+6 ms) 2024-12-04T20:16:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 49bf9bd65a1bae51a9870869671e6eee, disabling compactions & flushes 2024-12-04T20:16:46,008 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. after waiting 0 ms 2024-12-04T20:16:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:46,008 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:46,008 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 49bf9bd65a1bae51a9870869671e6eee: Waiting for close lock at 1733343406008Disabling compacts and flushes for region at 1733343406008Disabling writes for close at 1733343406008Writing region close event to WAL at 1733343406008Closed at 1733343406008 2024-12-04T20:16:46,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741853_1029 (size=175) 2024-12-04T20:16:46,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741853_1029 (size=175) 2024-12-04T20:16:46,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741853_1029 (size=175) 2024-12-04T20:16:46,015 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:46,015 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-65412602:(num 1733343405936) 2024-12-04T20:16:46,015 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:46,017 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:46,030 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, exclude list is [], retry=0 2024-12-04T20:16:46,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:46,033 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:46,034 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:46,057 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 2024-12-04T20:16:46,057 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:46,244 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, size=0 (0bytes) 2024-12-04T20:16:46,244 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 might be still open, length is 0 2024-12-04T20:16:46,244 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 2024-12-04T20:16:46,245 WARN [IPC Server handler 0 on default port 44433 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-04T20:16:46,245 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 after 1ms 2024-12-04T20:16:46,975 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:37070 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:42009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:37070 dst: /127.0.0.1:42009 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42009 remote=/127.0.0.1:37070]. Total timeout mills is 60000, 59241 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:46,976 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46216 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:38071:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46216 dst: /127.0.0.1:38071 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:46,976 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:40988 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37737:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40988 dst: /127.0.0.1:37737 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:46,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741855_1032 (size=497775) 2024-12-04T20:16:46,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741855_1032 (size=497775) 2024-12-04T20:16:46,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741855_1032 (size=497775) 2024-12-04T20:16:49,902 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:16:49,950 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-04T20:16:50,247 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 after 4003ms 2024-12-04T20:16:50,280 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:50,281 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 took 4037ms 2024-12-04T20:16:50,286 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733343406018.temp 2024-12-04T20:16:50,311 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp 2024-12-04T20:16:50,372 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018; continuing. 2024-12-04T20:16:50,372 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 so closing down 2024-12-04T20:16:50,372 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:50,373 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:50,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741856_1033 (size=497783) 2024-12-04T20:16:50,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741856_1033 (size=497783) 2024-12-04T20:16:50,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741856_1033 (size=497783) 2024-12-04T20:16:50,376 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp (wrote 3002 edits, skipped 0 edits in 45 ms) 2024-12-04T20:16:50,378 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 2024-12-04T20:16:50,379 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 96 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T20:16:50,379 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, journal: Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, size=0 (0bytes) at 1733343406244Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp at 1733343410311 (+4067 ms)Split 1024 edits, skipped 0 edits. at 1733343410328 (+17 ms)Split 2048 edits, skipped 0 edits. at 1733343410352 (+24 ms)Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 so closing down at 1733343410372 (+20 ms)3 split writer threads finished at 1733343410373 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp (wrote 3002 edits, skipped 0 edits in 45 ms) at 1733343410376 (+3 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000000001-wal.1733343406018.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 at 1733343410378 (+2 ms)Processed 3002 edits across 1 Regions in 96 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018, size=0, length=0, corrupted=false, cancelled=false at 1733343410379 (+1 ms) 2024-12-04T20:16:50,381 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343406018 2024-12-04T20:16:50,383 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 2024-12-04T20:16:50,383 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:50,385 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:50,401 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343410386, exclude list is [], retry=0 2024-12-04T20:16:50,403 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:50,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:50,404 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:50,428 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343410386 2024-12-04T20:16:50,428 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:50,428 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:50,430 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,431 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName a 2024-12-04T20:16:50,431 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:50,431 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:50,432 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,433 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName b 2024-12-04T20:16:50,433 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:50,433 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:50,433 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,434 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 49bf9bd65a1bae51a9870869671e6eee columnFamilyName c 2024-12-04T20:16:50,434 DEBUG [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:50,435 INFO [StoreOpener-49bf9bd65a1bae51a9870869671e6eee-1 {}] regionserver.HStore(327): Store=49bf9bd65a1bae51a9870869671e6eee/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:50,435 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,436 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,438 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:50,439 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 2024-12-04T20:16:50,462 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:50,491 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T20:16:50,794 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49bf9bd65a1bae51a9870869671e6eee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T20:16:50,829 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/9f9c6557bb924a9d8252b27fd1505698 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733343406068/Put/seqid=0 2024-12-04T20:16:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741858_1035 (size=50463) 2024-12-04T20:16:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741858_1035 (size=50463) 2024-12-04T20:16:50,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741858_1035 (size=50463) 2024-12-04T20:16:50,838 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/9f9c6557bb924a9d8252b27fd1505698 2024-12-04T20:16:50,846 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/9f9c6557bb924a9d8252b27fd1505698 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/a/9f9c6557bb924a9d8252b27fd1505698 2024-12-04T20:16:50,854 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/a/9f9c6557bb924a9d8252b27fd1505698, entries=754, sequenceid=754, filesize=49.3 K 2024-12-04T20:16:50,855 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 49bf9bd65a1bae51a9870869671e6eee in 61ms, sequenceid=754, compaction requested=false; wal=null 2024-12-04T20:16:50,870 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T20:16:50,870 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49bf9bd65a1bae51a9870869671e6eee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T20:16:50,879 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/e9709524e6c543db9fa34b88a49290e1 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733343406121/Put/seqid=0 2024-12-04T20:16:50,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741859_1036 (size=20072) 2024-12-04T20:16:50,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741859_1036 (size=20072) 2024-12-04T20:16:50,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741859_1036 (size=20072) 2024-12-04T20:16:50,889 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/e9709524e6c543db9fa34b88a49290e1 2024-12-04T20:16:50,915 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/0a00a01b3b7a4fa4b5d32dde9c40b2e8 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733343406144/Put/seqid=0 2024-12-04T20:16:50,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741860_1037 (size=35835) 2024-12-04T20:16:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741860_1037 (size=35835) 2024-12-04T20:16:50,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741860_1037 (size=35835) 2024-12-04T20:16:50,926 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/0a00a01b3b7a4fa4b5d32dde9c40b2e8 2024-12-04T20:16:50,935 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/a/e9709524e6c543db9fa34b88a49290e1 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/a/e9709524e6c543db9fa34b88a49290e1 2024-12-04T20:16:50,943 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/a/e9709524e6c543db9fa34b88a49290e1, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-04T20:16:50,945 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/0a00a01b3b7a4fa4b5d32dde9c40b2e8 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/b/0a00a01b3b7a4fa4b5d32dde9c40b2e8 2024-12-04T20:16:50,954 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/b/0a00a01b3b7a4fa4b5d32dde9c40b2e8, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-04T20:16:50,955 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 49bf9bd65a1bae51a9870869671e6eee in 84ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-04T20:16:50,973 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T20:16:50,974 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49bf9bd65a1bae51a9870869671e6eee 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-04T20:16:50,983 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/5779888ce0324a6cade082ab12679610 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733343406158/Put/seqid=0 2024-12-04T20:16:50,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741861_1038 (size=35082) 2024-12-04T20:16:50,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741861_1038 (size=35082) 2024-12-04T20:16:50,994 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741861_1038 (size=35082) 2024-12-04T20:16:50,995 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/5779888ce0324a6cade082ab12679610 2024-12-04T20:16:51,021 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/7d3d76beb3934c68a8decc1f6b5a55e7 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733343406181/Put/seqid=0 2024-12-04T20:16:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741862_1039 (size=20825) 2024-12-04T20:16:51,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741862_1039 (size=20825) 2024-12-04T20:16:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741862_1039 (size=20825) 2024-12-04T20:16:51,034 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/7d3d76beb3934c68a8decc1f6b5a55e7 2024-12-04T20:16:51,041 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/b/5779888ce0324a6cade082ab12679610 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/b/5779888ce0324a6cade082ab12679610 2024-12-04T20:16:51,047 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/b/5779888ce0324a6cade082ab12679610, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-04T20:16:51,048 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/7d3d76beb3934c68a8decc1f6b5a55e7 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/c/7d3d76beb3934c68a8decc1f6b5a55e7 2024-12-04T20:16:51,055 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/c/7d3d76beb3934c68a8decc1f6b5a55e7, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-04T20:16:51,055 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 49bf9bd65a1bae51a9870869671e6eee in 82ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-04T20:16:51,075 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733343406214/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:51,079 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 2024-12-04T20:16:51,080 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-04T20:16:51,080 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 49bf9bd65a1bae51a9870869671e6eee 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-04T20:16:51,090 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/5756efc0a5604eb48b9d28c8bc6ba378 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733343406187/Put/seqid=0 2024-12-04T20:16:51,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741863_1040 (size=50301) 2024-12-04T20:16:51,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741863_1040 (size=50301) 2024-12-04T20:16:51,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741863_1040 (size=50301) 2024-12-04T20:16:51,098 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/5756efc0a5604eb48b9d28c8bc6ba378 2024-12-04T20:16:51,104 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5756efc0a5604eb48b9d28c8bc6ba378 2024-12-04T20:16:51,106 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/.tmp/c/5756efc0a5604eb48b9d28c8bc6ba378 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/c/5756efc0a5604eb48b9d28c8bc6ba378 2024-12-04T20:16:51,115 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5756efc0a5604eb48b9d28c8bc6ba378 2024-12-04T20:16:51,115 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/c/5756efc0a5604eb48b9d28c8bc6ba378, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-04T20:16:51,115 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 49bf9bd65a1bae51a9870869671e6eee in 35ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-04T20:16:51,116 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/0000000000000003002 2024-12-04T20:16:51,117 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:51,117 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:51,118 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-04T20:16:51,120 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 49bf9bd65a1bae51a9870869671e6eee 2024-12-04T20:16:51,123 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenIntoWAL/49bf9bd65a1bae51a9870869671e6eee/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-04T20:16:51,124 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 49bf9bd65a1bae51a9870869671e6eee; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63564361, jitterRate=-0.0528172105550766}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-04T20:16:51,124 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 49bf9bd65a1bae51a9870869671e6eee: Writing region info on filesystem at 1733343410428Initializing all the Stores at 1733343410429 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343410429Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343410430 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343410430Cleaning up temporary data from old regions at 1733343411117 (+687 ms)Region opened successfully at 1733343411124 (+7 ms) 2024-12-04T20:16:51,178 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 49bf9bd65a1bae51a9870869671e6eee, disabling compactions & flushes 2024-12-04T20:16:51,178 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:51,178 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:51,178 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. after waiting 0 ms 2024-12-04T20:16:51,178 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:51,180 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733343405930.49bf9bd65a1bae51a9870869671e6eee. 2024-12-04T20:16:51,180 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 49bf9bd65a1bae51a9870869671e6eee: Waiting for close lock at 1733343411178Disabling compacts and flushes for region at 1733343411178Disabling writes for close at 1733343411178Writing region close event to WAL at 1733343411180 (+2 ms)Closed at 1733343411180 2024-12-04T20:16:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741857_1034 (size=175) 2024-12-04T20:16:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741857_1034 (size=175) 2024-12-04T20:16:51,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741857_1034 (size=175) 2024-12-04T20:16:51,186 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:51,187 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733343410386) 2024-12-04T20:16:51,202 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=399 (was 383) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44373 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-661794630_22 at /127.0.0.1:33896 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44433 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:44433 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-661794630_22 at /127.0.0.1:46752 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-661794630_22 at /127.0.0.1:33924 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37665 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-661794630_22 at /127.0.0.1:33926 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:44373 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:37665 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 138) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5271 (was 5370) 2024-12-04T20:16:51,217 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=399, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=5265 2024-12-04T20:16:51,238 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:51,242 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:51,243 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:51,247 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-82215586, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-82215586, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:51,260 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-82215586/hregion-82215586.1733343411248, exclude list is [], retry=0 2024-12-04T20:16:51,264 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:51,264 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:51,265 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:51,290 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-82215586/hregion-82215586.1733343411248 2024-12-04T20:16:51,291 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:51,291 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4e59fedf16f882979b190381058b7a9e, NAME => 'test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:16:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741865_1042 (size=43) 2024-12-04T20:16:51,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741865_1042 (size=43) 2024-12-04T20:16:51,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741865_1042 (size=43) 2024-12-04T20:16:51,302 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:51,304 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,305 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName a 2024-12-04T20:16:51,306 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:51,306 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:51,306 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,308 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName b 2024-12-04T20:16:51,308 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:51,309 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:51,309 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,311 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName c 2024-12-04T20:16:51,311 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:51,311 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:51,312 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,315 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,315 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:51,318 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:51,321 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:51,322 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e59fedf16f882979b190381058b7a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65608912, jitterRate=-0.02235102653503418}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:51,323 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e59fedf16f882979b190381058b7a9e: Writing region info on filesystem at 1733343411302Initializing all the Stores at 1733343411303 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343411303Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343411303Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343411303Cleaning up temporary data from old regions at 1733343411315 (+12 ms)Region opened successfully at 1733343411323 (+8 ms) 2024-12-04T20:16:51,324 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e59fedf16f882979b190381058b7a9e, disabling compactions & flushes 2024-12-04T20:16:51,324 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:51,324 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:51,324 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. after waiting 0 ms 2024-12-04T20:16:51,324 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:51,324 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:51,325 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e59fedf16f882979b190381058b7a9e: Waiting for close lock at 1733343411324Disabling compacts and flushes for region at 1733343411324Disabling writes for close at 1733343411324Writing region close event to WAL at 1733343411324Closed at 1733343411324 2024-12-04T20:16:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741864_1041 (size=175) 2024-12-04T20:16:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741864_1041 (size=175) 2024-12-04T20:16:51,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741864_1041 (size=175) 2024-12-04T20:16:51,330 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:51,330 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-82215586:(num 1733343411248) 2024-12-04T20:16:51,331 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:51,333 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:51,351 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, exclude list is [], retry=0 2024-12-04T20:16:51,354 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:51,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:51,355 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:51,380 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 2024-12-04T20:16:51,380 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:51,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741866_1043 (size=371388) 2024-12-04T20:16:51,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741866_1043 (size=371388) 2024-12-04T20:16:51,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741866_1043 (size=371388) 2024-12-04T20:16:51,529 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, size=362.7 K (371388bytes) 2024-12-04T20:16:51,529 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 2024-12-04T20:16:51,529 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 after 0ms 2024-12-04T20:16:51,555 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:51,557 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 took 28ms 2024-12-04T20:16:51,561 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733343411334.temp 2024-12-04T20:16:51,589 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp 2024-12-04T20:16:51,602 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 so closing down 2024-12-04T20:16:51,602 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:51,605 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:51,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741867_1044 (size=371388) 2024-12-04T20:16:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741867_1044 (size=371388) 2024-12-04T20:16:51,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741867_1044 (size=371388) 2024-12-04T20:16:51,611 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp (wrote 3000 edits, skipped 0 edits in 15 ms) 2024-12-04T20:16:51,613 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp to hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 2024-12-04T20:16:51,613 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-04T20:16:51,613 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, journal: Splitting hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, size=362.7 K (371388bytes) at 1733343411529Split 1024 edits, skipped 0 edits. at 1733343411574 (+45 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp at 1733343411589 (+15 ms)Split 2048 edits, skipped 0 edits. at 1733343411589Finishing writing output for hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 so closing down at 1733343411602 (+13 ms)3 split writer threads finished at 1733343411605 (+3 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp (wrote 3000 edits, skipped 0 edits in 15 ms) at 1733343411611 (+6 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000000001-wal.1733343411334.temp to hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 at 1733343411613 (+2 ms)Processed 3000 edits across 1 Regions in 56 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733343411613 2024-12-04T20:16:51,616 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411334 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343411334 2024-12-04T20:16:51,617 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 2024-12-04T20:16:51,617 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:51,620 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:51,633 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, exclude list is [], retry=0 2024-12-04T20:16:51,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:51,637 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:51,638 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:51,665 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 2024-12-04T20:16:51,666 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:51,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741868_1045 (size=371515) 2024-12-04T20:16:51,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741868_1045 (size=371515) 2024-12-04T20:16:51,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741868_1045 (size=371515) 2024-12-04T20:16:51,811 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, size=362.8 K (371515bytes) 2024-12-04T20:16:51,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 2024-12-04T20:16:51,811 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 after 0ms 2024-12-04T20:16:51,839 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:51,840 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 took 29ms 2024-12-04T20:16:51,844 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733343411621.temp 2024-12-04T20:16:51,893 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 so closing down 2024-12-04T20:16:51,893 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:51,911 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp 2024-12-04T20:16:51,931 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741869_1046 (size=371515) 2024-12-04T20:16:51,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741869_1046 (size=371515) 2024-12-04T20:16:51,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741869_1046 (size=371515) 2024-12-04T20:16:51,941 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp (wrote 3000 edits, skipped 0 edits in 18 ms) 2024-12-04T20:16:51,943 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp to hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 2024-12-04T20:16:51,943 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-04T20:16:51,943 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, journal: Splitting hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, size=362.8 K (371515bytes) at 1733343411811Split 1024 edits, skipped 0 edits. at 1733343411854 (+43 ms)Split 2048 edits, skipped 0 edits. at 1733343411883 (+29 ms)Finishing writing output for hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 so closing down at 1733343411893 (+10 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp at 1733343411911 (+18 ms)3 split writer threads finished at 1733343411931 (+20 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp (wrote 3000 edits, skipped 0 edits in 18 ms) at 1733343411941 (+10 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003001-wal.1733343411621.temp to hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 at 1733343411943 (+2 ms)Processed 3000 edits across 1 Regions in 102 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733343411943 2024-12-04T20:16:51,947 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411621 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343411621 2024-12-04T20:16:51,948 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 2024-12-04T20:16:51,949 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:51,951 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/test2727-manual,16010,1733343411238, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:51,964 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411952, exclude list is [], retry=0 2024-12-04T20:16:51,967 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:51,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:51,968 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:52,008 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411952 2024-12-04T20:16:52,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:16:52,009 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4e59fedf16f882979b190381058b7a9e, NAME => 'test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:52,009 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:52,009 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,010 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,012 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,013 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName a 2024-12-04T20:16:52,013 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,014 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,014 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,016 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName b 2024-12-04T20:16:52,016 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,016 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,017 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,018 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4e59fedf16f882979b190381058b7a9e columnFamilyName c 2024-12-04T20:16:52,018 DEBUG [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,018 INFO [StoreOpener-4e59fedf16f882979b190381058b7a9e-1 {}] regionserver.HStore(327): Store=4e59fedf16f882979b190381058b7a9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,019 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,020 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,023 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 2024-12-04T20:16:52,064 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:52,104 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 2024-12-04T20:16:52,105 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 2024-12-04T20:16:52,128 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:52,155 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 2024-12-04T20:16:52,156 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4e59fedf16f882979b190381058b7a9e 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-04T20:16:52,173 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/a/dd2f6a1558f64a09a9e0ff6410198b03 is 41, key is test2727/a:100/1733343411671/Put/seqid=0 2024-12-04T20:16:52,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741871_1048 (size=84227) 2024-12-04T20:16:52,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741871_1048 (size=84227) 2024-12-04T20:16:52,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741871_1048 (size=84227) 2024-12-04T20:16:52,180 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/a/dd2f6a1558f64a09a9e0ff6410198b03 2024-12-04T20:16:52,215 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/b/4bb18ad837b6443e927c771c0639e636 is 41, key is test2727/b:100/1733343411710/Put/seqid=0 2024-12-04T20:16:52,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741872_1049 (size=84609) 2024-12-04T20:16:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741872_1049 (size=84609) 2024-12-04T20:16:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741872_1049 (size=84609) 2024-12-04T20:16:52,220 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/b/4bb18ad837b6443e927c771c0639e636 2024-12-04T20:16:52,242 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/c/472f2808a42e45daa92fa9a721bd566d is 41, key is test2727/c:100/1733343411751/Put/seqid=0 2024-12-04T20:16:52,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741873_1050 (size=84609) 2024-12-04T20:16:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741873_1050 (size=84609) 2024-12-04T20:16:52,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741873_1050 (size=84609) 2024-12-04T20:16:52,248 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/c/472f2808a42e45daa92fa9a721bd566d 2024-12-04T20:16:52,255 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/a/dd2f6a1558f64a09a9e0ff6410198b03 as hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/a/dd2f6a1558f64a09a9e0ff6410198b03 2024-12-04T20:16:52,261 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/a/dd2f6a1558f64a09a9e0ff6410198b03, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-04T20:16:52,262 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/b/4bb18ad837b6443e927c771c0639e636 as hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/b/4bb18ad837b6443e927c771c0639e636 2024-12-04T20:16:52,269 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/b/4bb18ad837b6443e927c771c0639e636, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T20:16:52,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/.tmp/c/472f2808a42e45daa92fa9a721bd566d as hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/c/472f2808a42e45daa92fa9a721bd566d 2024-12-04T20:16:52,277 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/c/472f2808a42e45daa92fa9a721bd566d, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-04T20:16:52,277 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4e59fedf16f882979b190381058b7a9e in 121ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-04T20:16:52,278 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000003000 2024-12-04T20:16:52,279 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/0000000000000006000 2024-12-04T20:16:52,280 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,280 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,280 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:52,282 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4e59fedf16f882979b190381058b7a9e 2024-12-04T20:16:52,285 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/test2727/4e59fedf16f882979b190381058b7a9e/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-04T20:16:52,286 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4e59fedf16f882979b190381058b7a9e; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69510298, jitterRate=0.03578415513038635}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:52,287 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4e59fedf16f882979b190381058b7a9e: Writing region info on filesystem at 1733343412010Initializing all the Stores at 1733343412011 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412011Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412011Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412011Obtaining lock to block concurrent updates at 1733343412156 (+145 ms)Preparing flush snapshotting stores in 4e59fedf16f882979b190381058b7a9e at 1733343412156Finished memstore snapshotting test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733343412156Flushing stores of test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. at 1733343412156Flushing 4e59fedf16f882979b190381058b7a9e/a: creating writer at 1733343412156Flushing 4e59fedf16f882979b190381058b7a9e/a: appending metadata at 1733343412172 (+16 ms)Flushing 4e59fedf16f882979b190381058b7a9e/a: closing flushed file at 1733343412172Flushing 4e59fedf16f882979b190381058b7a9e/b: creating writer at 1733343412187 (+15 ms)Flushing 4e59fedf16f882979b190381058b7a9e/b: appending metadata at 1733343412213 (+26 ms)Flushing 4e59fedf16f882979b190381058b7a9e/b: closing flushed file at 1733343412213Flushing 4e59fedf16f882979b190381058b7a9e/c: creating writer at 1733343412225 (+12 ms)Flushing 4e59fedf16f882979b190381058b7a9e/c: appending metadata at 1733343412241 (+16 ms)Flushing 4e59fedf16f882979b190381058b7a9e/c: closing flushed file at 1733343412241Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6cd3f153: reopening flushed file at 1733343412253 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@343658eb: reopening flushed file at 1733343412261 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@110c2cc1: reopening flushed file at 1733343412269 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 4e59fedf16f882979b190381058b7a9e in 121ms, sequenceid=6000, compaction requested=false; wal=null at 1733343412277 (+8 ms)Cleaning up temporary data from old regions at 1733343412280 (+3 ms)Region opened successfully at 1733343412287 (+7 ms) 2024-12-04T20:16:52,288 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-04T20:16:52,288 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4e59fedf16f882979b190381058b7a9e, disabling compactions & flushes 2024-12-04T20:16:52,288 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:52,288 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:52,288 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. after waiting 0 ms 2024-12-04T20:16:52,288 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:52,289 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733343411240.4e59fedf16f882979b190381058b7a9e. 2024-12-04T20:16:52,289 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4e59fedf16f882979b190381058b7a9e: Waiting for close lock at 1733343412288Disabling compacts and flushes for region at 1733343412288Disabling writes for close at 1733343412288Writing region close event to WAL at 1733343412289 (+1 ms)Closed at 1733343412289 2024-12-04T20:16:52,294 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733343411238/wal.1733343411952 not finished, retry = 0 2024-12-04T20:16:52,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741870_1047 (size=175) 2024-12-04T20:16:52,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741870_1047 (size=175) 2024-12-04T20:16:52,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741870_1047 (size=175) 2024-12-04T20:16:52,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-04T20:16:52,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-04T20:16:52,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:16:52,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-04T20:16:52,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-04T20:16:52,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-04T20:16:52,304 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-04T20:16:52,304 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-04T20:16:52,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:52,399 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733343411952) 2024-12-04T20:16:52,416 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401 (was 399) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46896 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:33924 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:33926 [Waiting for operation #23] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=927 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=5040 (was 5265) 2024-12-04T20:16:52,427 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=927, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=5039 2024-12-04T20:16:52,444 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:52,450 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:52,450 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733343412450 2024-12-04T20:16:52,478 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 2024-12-04T20:16:52,480 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:16:52,481 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7963a5cbcd022768d8c888c2873be7a2, NAME => 'testSequentialEditLogSeqNum,,1733343412444.7963a5cbcd022768d8c888c2873be7a2.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:52,482 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733343412444.7963a5cbcd022768d8c888c2873be7a2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:52,482 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,482 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,483 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2 doesn't exist for region: 7963a5cbcd022768d8c888c2873be7a2 on table testSequentialEditLogSeqNum 2024-12-04T20:16:52,483 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 7963a5cbcd022768d8c888c2873be7a2 on table testSequentialEditLogSeqNum 2024-12-04T20:16:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741875_1052 (size=62) 2024-12-04T20:16:52,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741875_1052 (size=62) 2024-12-04T20:16:52,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741875_1052 (size=62) 2024-12-04T20:16:52,494 INFO [StoreOpener-7963a5cbcd022768d8c888c2873be7a2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,496 INFO [StoreOpener-7963a5cbcd022768d8c888c2873be7a2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7963a5cbcd022768d8c888c2873be7a2 columnFamilyName a 2024-12-04T20:16:52,496 DEBUG [StoreOpener-7963a5cbcd022768d8c888c2873be7a2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,497 INFO [StoreOpener-7963a5cbcd022768d8c888c2873be7a2-1 {}] regionserver.HStore(327): Store=7963a5cbcd022768d8c888c2873be7a2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,497 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,498 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,498 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,498 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,500 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7963a5cbcd022768d8c888c2873be7a2 2024-12-04T20:16:52,502 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:52,503 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7963a5cbcd022768d8c888c2873be7a2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60517024, jitterRate=-0.09822607040405273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:16:52,503 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7963a5cbcd022768d8c888c2873be7a2: Writing region info on filesystem at 1733343412482Initializing all the Stores at 1733343412493 (+11 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412493Cleaning up temporary data from old regions at 1733343412498 (+5 ms)Region opened successfully at 1733343412503 (+5 ms) 2024-12-04T20:16:52,513 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7963a5cbcd022768d8c888c2873be7a2 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-04T20:16:52,532 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/.tmp/a/d4ab552c4f574692b177b416a9f0af8d is 81, key is testSequentialEditLogSeqNum/a:x0/1733343412503/Put/seqid=0 2024-12-04T20:16:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741876_1053 (size=5833) 2024-12-04T20:16:52,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741876_1053 (size=5833) 2024-12-04T20:16:52,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741876_1053 (size=5833) 2024-12-04T20:16:52,540 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/.tmp/a/d4ab552c4f574692b177b416a9f0af8d 2024-12-04T20:16:52,548 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/.tmp/a/d4ab552c4f574692b177b416a9f0af8d as hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/a/d4ab552c4f574692b177b416a9f0af8d 2024-12-04T20:16:52,555 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/a/d4ab552c4f574692b177b416a9f0af8d, entries=10, sequenceid=13, filesize=5.7 K 2024-12-04T20:16:52,557 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 7963a5cbcd022768d8c888c2873be7a2 in 44ms, sequenceid=13, compaction requested=false 2024-12-04T20:16:52,557 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 7963a5cbcd022768d8c888c2873be7a2: 2024-12-04T20:16:52,563 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:16:52,563 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:16:52,563 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:16:52,564 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:16:52,564 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-04T20:16:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741874_1051 (size=3256) 2024-12-04T20:16:52,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741874_1051 (size=3256) 2024-12-04T20:16:52,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741874_1051 (size=3256) 2024-12-04T20:16:52,585 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450, size=3.2 K (3256bytes) 2024-12-04T20:16:52,585 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 2024-12-04T20:16:52,586 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 after 1ms 2024-12-04T20:16:52,617 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:52,618 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 took 33ms 2024-12-04T20:16:52,621 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 so closing down 2024-12-04T20:16:52,621 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:52,622 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733343412450.temp 2024-12-04T20:16:52,647 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp 2024-12-04T20:16:52,648 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:52,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741877_1054 (size=2650) 2024-12-04T20:16:52,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741877_1054 (size=2650) 2024-12-04T20:16:52,657 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741877_1054 (size=2650) 2024-12-04T20:16:52,658 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:52,659 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp to hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000020 2024-12-04T20:16:52,659 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-04T20:16:52,659 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450, journal: Splitting hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450, size=3.2 K (3256bytes) at 1733343412585Finishing writing output for hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450 so closing down at 1733343412621 (+36 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp at 1733343412647 (+26 ms)3 split writer threads finished at 1733343412648 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733343412658 (+10 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000003-wal.1733343412450.temp to hdfs://localhost:44433/hbase/data/default/testSequentialEditLogSeqNum/7963a5cbcd022768d8c888c2873be7a2/recovered.edits/0000000000000000020 at 1733343412659 (+1 ms)Processed 17 edits across 1 Regions in 41 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733343412443/wal.1733343412450, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733343412659 2024-12-04T20:16:52,672 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46896 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:33926 [Waiting for operation #27] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=963 (was 927) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=172 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=5032 (was 5039) 2024-12-04T20:16:52,688 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=963, MaxFileDescriptor=1048576, SystemLoadAverage=172, ProcessCount=11, AvailableMemoryMB=5030 2024-12-04T20:16:52,710 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:52,714 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:52,737 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:16:52,740 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-79003272, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-79003272, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:52,754 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-79003272/hregion-79003272.1733343412741, exclude list is [], retry=0 2024-12-04T20:16:52,757 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:52,757 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:52,757 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:52,784 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-79003272/hregion-79003272.1733343412741 2024-12-04T20:16:52,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:52,785 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => b8b5f62f9adb41760c6a48c1bb6dcb55, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:16:52,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741879_1056 (size=70) 2024-12-04T20:16:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741879_1056 (size=70) 2024-12-04T20:16:52,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741879_1056 (size=70) 2024-12-04T20:16:52,795 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:52,797 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,798 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName a 2024-12-04T20:16:52,798 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,799 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,799 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,800 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName b 2024-12-04T20:16:52,800 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,800 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,801 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,802 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName c 2024-12-04T20:16:52,802 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,802 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,802 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,803 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,804 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,804 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,805 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:52,806 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,808 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:52,809 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b8b5f62f9adb41760c6a48c1bb6dcb55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67736482, jitterRate=0.009352236986160278}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:52,809 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b8b5f62f9adb41760c6a48c1bb6dcb55: Writing region info on filesystem at 1733343412795Initializing all the Stores at 1733343412796 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412796Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412797 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412797Cleaning up temporary data from old regions at 1733343412804 (+7 ms)Region opened successfully at 1733343412809 (+5 ms) 2024-12-04T20:16:52,810 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b8b5f62f9adb41760c6a48c1bb6dcb55, disabling compactions & flushes 2024-12-04T20:16:52,810 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:52,810 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:52,810 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. after waiting 0 ms 2024-12-04T20:16:52,810 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:52,810 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:52,810 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b8b5f62f9adb41760c6a48c1bb6dcb55: Waiting for close lock at 1733343412810Disabling compacts and flushes for region at 1733343412810Disabling writes for close at 1733343412810Writing region close event to WAL at 1733343412810Closed at 1733343412810 2024-12-04T20:16:52,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741878_1055 (size=175) 2024-12-04T20:16:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741878_1055 (size=175) 2024-12-04T20:16:52,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741878_1055 (size=175) 2024-12-04T20:16:52,817 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:52,817 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-79003272:(num 1733343412741) 2024-12-04T20:16:52,817 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:52,819 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:52,831 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, exclude list is [], retry=0 2024-12-04T20:16:52,834 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:52,834 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:52,834 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:52,859 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 2024-12-04T20:16:52,859 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:16:52,859 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b8b5f62f9adb41760c6a48c1bb6dcb55, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:52,859 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:52,859 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,860 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,861 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,862 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName a 2024-12-04T20:16:52,862 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,862 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,863 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,863 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName b 2024-12-04T20:16:52,863 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,864 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,864 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,865 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName c 2024-12-04T20:16:52,865 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:52,865 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:52,865 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,866 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,867 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,868 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,868 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,869 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:52,870 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:52,871 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b8b5f62f9adb41760c6a48c1bb6dcb55; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72803691, jitterRate=0.08485953509807587}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:52,872 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b8b5f62f9adb41760c6a48c1bb6dcb55: Writing region info on filesystem at 1733343412860Initializing all the Stores at 1733343412861 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412861Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412861Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343412861Cleaning up temporary data from old regions at 1733343412869 (+8 ms)Region opened successfully at 1733343412872 (+3 ms) 2024-12-04T20:16:52,875 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733343412874/Put/seqid=0 2024-12-04T20:16:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741881_1058 (size=4826) 2024-12-04T20:16:52,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741881_1058 (size=4826) 2024-12-04T20:16:52,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741881_1058 (size=4826) 2024-12-04T20:16:52,883 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44433/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in b8b5f62f9adb41760c6a48c1bb6dcb55/a 2024-12-04T20:16:52,892 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-04T20:16:52,892 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T20:16:52,892 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for b8b5f62f9adb41760c6a48c1bb6dcb55: 2024-12-04T20:16:52,894 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/34c206f02f0e4d79ba6c527528614e89_SeqId_3_ 2024-12-04T20:16:52,895 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44433/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b8b5f62f9adb41760c6a48c1bb6dcb55/a as hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/34c206f02f0e4d79ba6c527528614e89_SeqId_3_ - updating store file list. 2024-12-04T20:16:52,900 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 34c206f02f0e4d79ba6c527528614e89_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T20:16:52,901 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/34c206f02f0e4d79ba6c527528614e89_SeqId_3_ into b8b5f62f9adb41760c6a48c1bb6dcb55/a 2024-12-04T20:16:52,901 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44433/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into b8b5f62f9adb41760c6a48c1bb6dcb55/a (new location: hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/34c206f02f0e4d79ba6c527528614e89_SeqId_3_) 2024-12-04T20:16:52,935 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, size=0 (0bytes) 2024-12-04T20:16:52,935 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 might be still open, length is 0 2024-12-04T20:16:52,936 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 2024-12-04T20:16:52,936 WARN [IPC Server handler 1 on default port 44433 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-04T20:16:52,936 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 after 0ms 2024-12-04T20:16:55,862 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:34068 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:42009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34068 dst: /127.0.0.1:42009 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42009 remote=/127.0.0.1:34068]. Total timeout mills is 60000, 57045 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:55,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:46980 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37737:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46980 dst: /127.0.0.1:37737 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:55,863 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:34146 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:38071:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34146 dst: /127.0.0.1:38071 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:55,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741880_1059 (size=606) 2024-12-04T20:16:55,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741880_1059 (size=606) 2024-12-04T20:16:56,938 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 after 4001ms 2024-12-04T20:16:56,970 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:56,970 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 took 4035ms 2024-12-04T20:16:56,972 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820; continuing. 2024-12-04T20:16:56,972 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 so closing down 2024-12-04T20:16:56,973 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:56,974 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733343412820.temp 2024-12-04T20:16:56,998 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp 2024-12-04T20:16:56,999 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:57,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741882_1060 (size=329) 2024-12-04T20:16:57,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741882_1060 (size=329) 2024-12-04T20:16:57,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741882_1060 (size=329) 2024-12-04T20:16:57,005 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-04T20:16:57,007 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp to hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 2024-12-04T20:16:57,007 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T20:16:57,007 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, journal: Splitting hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, size=0 (0bytes) at 1733343412935Finishing writing output for hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 so closing down at 1733343416973 (+4038 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp at 1733343416998 (+25 ms)3 split writer threads finished at 1733343416999 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733343417005 (+6 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005-wal.1733343412820.temp to hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 at 1733343417007 (+2 ms)Processed 2 edits across 1 Regions in 36 ms; skipped=1; WAL=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820, size=0, length=0, corrupted=false, cancelled=false at 1733343417007 2024-12-04T20:16:57,008 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343412820 2024-12-04T20:16:57,009 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 2024-12-04T20:16:57,010 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:16:57,011 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:16:57,022 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343417012, exclude list is [], retry=0 2024-12-04T20:16:57,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:16:57,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:16:57,025 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:16:57,047 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343417012 2024-12-04T20:16:57,048 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:16:57,048 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => b8b5f62f9adb41760c6a48c1bb6dcb55, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:57,048 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:57,048 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,048 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,050 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,051 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName a 2024-12-04T20:16:57,051 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,057 DEBUG [StoreFileOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 34c206f02f0e4d79ba6c527528614e89_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-04T20:16:57,057 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/34c206f02f0e4d79ba6c527528614e89_SeqId_3_ 2024-12-04T20:16:57,057 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,057 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,058 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName b 2024-12-04T20:16:57,058 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,059 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,059 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,060 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region b8b5f62f9adb41760c6a48c1bb6dcb55 columnFamilyName c 2024-12-04T20:16:57,060 DEBUG [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,061 INFO [StoreOpener-b8b5f62f9adb41760c6a48c1bb6dcb55-1 {}] regionserver.HStore(327): Store=b8b5f62f9adb41760c6a48c1bb6dcb55/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,061 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,062 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,064 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,065 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 2024-12-04T20:16:57,088 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:57,090 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 2024-12-04T20:16:57,090 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing b8b5f62f9adb41760c6a48c1bb6dcb55 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-04T20:16:57,102 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/.tmp/a/b70e5459073d46de963c2170f12dc7a8 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733343412905/Put/seqid=0 2024-12-04T20:16:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741884_1062 (size=5149) 2024-12-04T20:16:57,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741884_1062 (size=5149) 2024-12-04T20:16:57,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741884_1062 (size=5149) 2024-12-04T20:16:57,109 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/.tmp/a/b70e5459073d46de963c2170f12dc7a8 2024-12-04T20:16:57,115 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/.tmp/a/b70e5459073d46de963c2170f12dc7a8 as hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/b70e5459073d46de963c2170f12dc7a8 2024-12-04T20:16:57,120 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/a/b70e5459073d46de963c2170f12dc7a8, entries=1, sequenceid=5, filesize=5.0 K 2024-12-04T20:16:57,120 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b8b5f62f9adb41760c6a48c1bb6dcb55 in 30ms, sequenceid=5, compaction requested=false; wal=null 2024-12-04T20:16:57,121 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/0000000000000000005 2024-12-04T20:16:57,123 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,123 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,124 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:16:57,125 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for b8b5f62f9adb41760c6a48c1bb6dcb55 2024-12-04T20:16:57,128 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/b8b5f62f9adb41760c6a48c1bb6dcb55/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-04T20:16:57,129 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened b8b5f62f9adb41760c6a48c1bb6dcb55; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75193149, jitterRate=0.1204652339220047}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:16:57,129 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for b8b5f62f9adb41760c6a48c1bb6dcb55: Writing region info on filesystem at 1733343417048Initializing all the Stores at 1733343417049 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343417049Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343417050 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343417050Obtaining lock to block concurrent updates at 1733343417090 (+40 ms)Preparing flush snapshotting stores in b8b5f62f9adb41760c6a48c1bb6dcb55 at 1733343417090Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733343417090Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. at 1733343417090Flushing b8b5f62f9adb41760c6a48c1bb6dcb55/a: creating writer at 1733343417090Flushing b8b5f62f9adb41760c6a48c1bb6dcb55/a: appending metadata at 1733343417102 (+12 ms)Flushing b8b5f62f9adb41760c6a48c1bb6dcb55/a: closing flushed file at 1733343417102Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cf24a93: reopening flushed file at 1733343417114 (+12 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for b8b5f62f9adb41760c6a48c1bb6dcb55 in 30ms, sequenceid=5, compaction requested=false; wal=null at 1733343417120 (+6 ms)Cleaning up temporary data from old regions at 1733343417123 (+3 ms)Region opened successfully at 1733343417129 (+6 ms) 2024-12-04T20:16:57,133 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing b8b5f62f9adb41760c6a48c1bb6dcb55, disabling compactions & flushes 2024-12-04T20:16:57,133 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:57,133 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:57,134 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. after waiting 0 ms 2024-12-04T20:16:57,134 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:57,135 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733343412711.b8b5f62f9adb41760c6a48c1bb6dcb55. 2024-12-04T20:16:57,135 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for b8b5f62f9adb41760c6a48c1bb6dcb55: Waiting for close lock at 1733343417133Disabling compacts and flushes for region at 1733343417133Disabling writes for close at 1733343417134 (+1 ms)Writing region close event to WAL at 1733343417135 (+1 ms)Closed at 1733343417135 2024-12-04T20:16:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741883_1061 (size=175) 2024-12-04T20:16:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741883_1061 (size=175) 2024-12-04T20:16:57,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741883_1061 (size=175) 2024-12-04T20:16:57,140 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:16:57,140 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733343417012) 2024-12-04T20:16:57,152 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=408 (was 406) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45181 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44433 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:45181 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-114879073_22 at /127.0.0.1:34118 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-114879073_22 at /127.0.0.1:47018 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:44433 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-114879073_22 at /127.0.0.1:34158 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 963) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 172), ProcessCount=11 (was 11), AvailableMemoryMB=5024 (was 5030) 2024-12-04T20:16:57,162 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=408, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=5023 2024-12-04T20:16:57,178 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:16:57,182 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-04T20:16:57,186 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is c2ef38372881,37947,1733343401962 2024-12-04T20:16:57,188 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@64401c65 2024-12-04T20:16:57,189 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-04T20:16:57,190 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51564, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-04T20:16:57,194 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-04T20:16:57,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T20:16:57,203 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-04T20:16:57,205 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-04T20:16:57,206 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,208 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-04T20:16:57,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:16:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741885_1063 (size=694) 2024-12-04T20:16:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741885_1063 (size=694) 2024-12-04T20:16:57,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741885_1063 (size=694) 2024-12-04T20:16:57,223 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061 2024-12-04T20:16:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741886_1064 (size=77) 2024-12-04T20:16:57,231 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741886_1064 (size=77) 2024-12-04T20:16:57,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741886_1064 (size=77) 2024-12-04T20:16:57,233 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:57,234 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:57,234 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,234 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,234 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:57,234 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,234 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,234 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343417233Disabling compacts and flushes for region at 1733343417233Disabling writes for close at 1733343417234 (+1 ms)Writing region close event to WAL at 1733343417234Closed at 1733343417234 2024-12-04T20:16:57,236 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-04T20:16:57,242 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733343417236"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733343417236"}]},"ts":"1733343417236"} 2024-12-04T20:16:57,246 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-04T20:16:57,248 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-04T20:16:57,251 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343417248"}]},"ts":"1733343417248"} 2024-12-04T20:16:57,255 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-04T20:16:57,256 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {c2ef38372881=0} racks are {/default-rack=0} 2024-12-04T20:16:57,257 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T20:16:57,257 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T20:16:57,257 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-04T20:16:57,258 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T20:16:57,258 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T20:16:57,258 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-04T20:16:57,258 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T20:16:57,258 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T20:16:57,258 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-04T20:16:57,258 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T20:16:57,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN}] 2024-12-04T20:16:57,261 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN 2024-12-04T20:16:57,263 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN; state=OFFLINE, location=c2ef38372881,39001,1733343402624; forceNewPlan=false, retain=false 2024-12-04T20:16:57,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:16:57,416 INFO [c2ef38372881:37947 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T20:16:57,416 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPENING, regionLocation=c2ef38372881,39001,1733343402624 2024-12-04T20:16:57,420 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN because future has completed 2024-12-04T20:16:57,421 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624}] 2024-12-04T20:16:57,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:16:57,574 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:16:57,576 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60887, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:16:57,584 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,585 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:57,586 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,586 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:57,586 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,586 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,589 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,592 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:16:57,592 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,593 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,594 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,597 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:16:57,597 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,597 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,597 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,598 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,599 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,599 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,599 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,600 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T20:16:57,601 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,603 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:16:57,604 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 398cfc7d795642642bde6a262384a0b3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66876450, jitterRate=-0.003463238477706909}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T20:16:57,604 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,605 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 398cfc7d795642642bde6a262384a0b3: Running coprocessor pre-open hook at 1733343417586Writing region info on filesystem at 1733343417586Initializing all the Stores at 1733343417588 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343417588Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343417589 (+1 ms)Cleaning up temporary data from old regions at 1733343417599 (+10 ms)Running coprocessor post-open hooks at 1733343417604 (+5 ms)Region opened successfully at 1733343417605 (+1 ms) 2024-12-04T20:16:57,606 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., pid=6, masterSystemTime=1733343417574 2024-12-04T20:16:57,610 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,610 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPEN, openSeqNum=2, regionLocation=c2ef38372881,39001,1733343402624 2024-12-04T20:16:57,610 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 because future has completed 2024-12-04T20:16:57,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-04T20:16:57,617 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 in 193 msec 2024-12-04T20:16:57,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-04T20:16:57,620 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN in 358 msec 2024-12-04T20:16:57,621 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-04T20:16:57,621 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733343417621"}]},"ts":"1733343417621"} 2024-12-04T20:16:57,624 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-04T20:16:57,625 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-04T20:16:57,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 429 msec 2024-12-04T20:16:57,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-04T20:16:57,843 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-04T20:16:57,843 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-04T20:16:57,845 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:16:57,850 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-04T20:16:57,851 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-04T20:16:57,852 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-04T20:16:57,865 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=2] 2024-12-04T20:16:57,866 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:16:57,868 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58636, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:16:57,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=398cfc7d795642642bde6a262384a0b3, source=c2ef38372881,39001,1733343402624, destination=c2ef38372881,45313,1733343402534, warming up region on c2ef38372881,45313,1733343402534 2024-12-04T20:16:57,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:16:57,887 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=398cfc7d795642642bde6a262384a0b3, source=c2ef38372881,39001,1733343402624, destination=c2ef38372881,45313,1733343402534, running balancer 2024-12-04T20:16:57,889 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34379, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:16:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE 2024-12-04T20:16:57,889 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE 2024-12-04T20:16:57,892 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=CLOSING, regionLocation=c2ef38372881,39001,1733343402624 2024-12-04T20:16:57,893 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(7855): Warmup {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:57,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:57,894 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,895 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:16:57,895 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,896 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,896 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:57,897 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:16:57,897 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:57,898 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:57,898 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE because future has completed 2024-12-04T20:16:57,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:57,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:57,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343417898Disabling compacts and flushes for region at 1733343417898Disabling writes for close at 1733343417899 (+1 ms)Writing region close event to WAL at 1733343417899Closed at 1733343417899 2024-12-04T20:16:57,899 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:16:57,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624}] 2024-12-04T20:16:57,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-04T20:16:58,059 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,059 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:16:58,060 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:58,061 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,061 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,061 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:58,061 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,061 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 398cfc7d795642642bde6a262384a0b3 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-04T20:16:58,082 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/1d30bcdd9ee245ed9368c8f03860cafc is 35, key is r1/cf1:q/1733343417869/Put/seqid=0 2024-12-04T20:16:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741887_1065 (size=4783) 2024-12-04T20:16:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741887_1065 (size=4783) 2024-12-04T20:16:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741887_1065 (size=4783) 2024-12-04T20:16:58,090 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/1d30bcdd9ee245ed9368c8f03860cafc 2024-12-04T20:16:58,097 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/1d30bcdd9ee245ed9368c8f03860cafc as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc 2024-12-04T20:16:58,104 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc, entries=1, sequenceid=5, filesize=4.7 K 2024-12-04T20:16:58,105 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 398cfc7d795642642bde6a262384a0b3 in 44ms, sequenceid=5, compaction requested=false 2024-12-04T20:16:58,105 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-04T20:16:58,110 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T20:16:58,112 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,112 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343418060Running coprocessor pre-close hooks at 1733343418060Disabling compacts and flushes for region at 1733343418060Disabling writes for close at 1733343418061 (+1 ms)Obtaining lock to block concurrent updates at 1733343418061Preparing flush snapshotting stores in 398cfc7d795642642bde6a262384a0b3 at 1733343418061Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733343418062 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. at 1733343418063 (+1 ms)Flushing 398cfc7d795642642bde6a262384a0b3/cf1: creating writer at 1733343418063Flushing 398cfc7d795642642bde6a262384a0b3/cf1: appending metadata at 1733343418082 (+19 ms)Flushing 398cfc7d795642642bde6a262384a0b3/cf1: closing flushed file at 1733343418082Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d6500ee: reopening flushed file at 1733343418096 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 398cfc7d795642642bde6a262384a0b3 in 44ms, sequenceid=5, compaction requested=false at 1733343418105 (+9 ms)Writing region close event to WAL at 1733343418106 (+1 ms)Running coprocessor post-close hooks at 1733343418110 (+4 ms)Closed at 1733343418112 (+2 ms) 2024-12-04T20:16:58,113 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 398cfc7d795642642bde6a262384a0b3 move to c2ef38372881,45313,1733343402534 record at close sequenceid=5 2024-12-04T20:16:58,115 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,116 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=CLOSED 2024-12-04T20:16:58,119 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 because future has completed 2024-12-04T20:16:58,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-04T20:16:58,124 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 in 220 msec 2024-12-04T20:16:58,126 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE; state=CLOSED, location=c2ef38372881,45313,1733343402534; forceNewPlan=false, retain=false 2024-12-04T20:16:58,276 INFO [c2ef38372881:37947 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T20:16:58,277 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPENING, regionLocation=c2ef38372881,45313,1733343402534 2024-12-04T20:16:58,279 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE because future has completed 2024-12-04T20:16:58,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534}] 2024-12-04T20:16:58,437 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,437 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:58,438 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,438 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:58,438 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,438 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,440 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,441 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:16:58,441 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:58,449 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc 2024-12-04T20:16:58,449 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:58,450 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,451 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:16:58,451 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:58,452 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:58,452 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,453 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,454 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,455 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,455 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,456 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T20:16:58,457 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,458 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 398cfc7d795642642bde6a262384a0b3; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67154494, jitterRate=6.799399852752686E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T20:16:58,458 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,459 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 398cfc7d795642642bde6a262384a0b3: Running coprocessor pre-open hook at 1733343418438Writing region info on filesystem at 1733343418438Initializing all the Stores at 1733343418439 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343418439Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343418440 (+1 ms)Cleaning up temporary data from old regions at 1733343418455 (+15 ms)Running coprocessor post-open hooks at 1733343418458 (+3 ms)Region opened successfully at 1733343418459 (+1 ms) 2024-12-04T20:16:58,460 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., pid=9, masterSystemTime=1733343418433 2024-12-04T20:16:58,463 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,463 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,464 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPEN, openSeqNum=9, regionLocation=c2ef38372881,45313,1733343402534 2024-12-04T20:16:58,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 because future has completed 2024-12-04T20:16:58,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-04T20:16:58,471 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 in 188 msec 2024-12-04T20:16:58,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE in 584 msec 2024-12-04T20:16:58,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:16:58,498 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40302, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:16:58,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:58636 deadline: 1733343478502, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=45313 startCode=1733343402534. As of locationSeqNum=5. 2024-12-04T20:16:58,509 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=45313 startCode=1733343402534. As of locationSeqNum=5. 2024-12-04T20:16:58,510 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=45313 startCode=1733343402534. As of locationSeqNum=5. 2024-12-04T20:16:58,510 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=45313 startCode=1733343402534. As of locationSeqNum=5. 2024-12-04T20:16:58,622 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-04T20:16:58,623 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40310, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-04T20:16:58,636 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 398cfc7d795642642bde6a262384a0b3 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-04T20:16:58,650 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/f4f44112de3c41a0aec4b6bc926c529b is 29, key is r1/cf1:/1733343418624/DeleteFamily/seqid=0 2024-12-04T20:16:58,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741888_1066 (size=4906) 2024-12-04T20:16:58,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741888_1066 (size=4906) 2024-12-04T20:16:58,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741888_1066 (size=4906) 2024-12-04T20:16:58,659 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,665 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,681 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf2/df44c659d0e84dbab5ae9e65e42e4d4f is 29, key is r1/cf2:/1733343418624/DeleteFamily/seqid=0 2024-12-04T20:16:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741889_1067 (size=4906) 2024-12-04T20:16:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741889_1067 (size=4906) 2024-12-04T20:16:58,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741889_1067 (size=4906) 2024-12-04T20:16:58,687 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf2/df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,692 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,693 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/f4f44112de3c41a0aec4b6bc926c529b as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,699 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,699 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T20:16:58,700 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf2/df44c659d0e84dbab5ae9e65e42e4d4f as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,706 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,706 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f, entries=1, sequenceid=12, filesize=4.8 K 2024-12-04T20:16:58,707 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 398cfc7d795642642bde6a262384a0b3 in 71ms, sequenceid=12, compaction requested=false 2024-12-04T20:16:58,708 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 398cfc7d795642642bde6a262384a0b3: 2024-12-04T20:16:58,710 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-04T20:16:58,712 DEBUG [Time-limited test {}] regionserver.HStore(1541): 398cfc7d795642642bde6a262384a0b3/cf1 is initiating major compaction (all files) 2024-12-04T20:16:58,712 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:16:58,712 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:58,713 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 398cfc7d795642642bde6a262384a0b3/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,713 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc, hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b] into tmpdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp, totalSize=9.5 K 2024-12-04T20:16:58,715 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1d30bcdd9ee245ed9368c8f03860cafc, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733343417869 2024-12-04T20:16:58,716 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting f4f44112de3c41a0aec4b6bc926c529b, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T20:16:58,730 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 398cfc7d795642642bde6a262384a0b3#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:16:58,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741890_1068 (size=4626) 2024-12-04T20:16:58,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741890_1068 (size=4626) 2024-12-04T20:16:58,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741890_1068 (size=4626) 2024-12-04T20:16:58,747 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf1/e13116986a554fda865d8a8a647339fa as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/e13116986a554fda865d8a8a647339fa 2024-12-04T20:16:58,762 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 398cfc7d795642642bde6a262384a0b3/cf1 of 398cfc7d795642642bde6a262384a0b3 into e13116986a554fda865d8a8a647339fa(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:16:58,762 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 398cfc7d795642642bde6a262384a0b3: 2024-12-04T20:16:58,762 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-04T20:16:58,762 DEBUG [Time-limited test {}] regionserver.HStore(1541): 398cfc7d795642642bde6a262384a0b3/cf2 is initiating major compaction (all files) 2024-12-04T20:16:58,762 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-04T20:16:58,762 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-04T20:16:58,762 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 398cfc7d795642642bde6a262384a0b3/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,762 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f] into tmpdir=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp, totalSize=4.8 K 2024-12-04T20:16:58,763 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting df44c659d0e84dbab5ae9e65e42e4d4f, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-04T20:16:58,770 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 398cfc7d795642642bde6a262384a0b3#cf2#compaction#17 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-04T20:16:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741891_1069 (size=4592) 2024-12-04T20:16:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741891_1069 (size=4592) 2024-12-04T20:16:58,777 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741891_1069 (size=4592) 2024-12-04T20:16:58,788 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/.tmp/cf2/a75d138285a44174bde94b30d82c6843 as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/a75d138285a44174bde94b30d82c6843 2024-12-04T20:16:58,796 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 398cfc7d795642642bde6a262384a0b3/cf2 of 398cfc7d795642642bde6a262384a0b3 into a75d138285a44174bde94b30d82c6843(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:16:58,796 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 398cfc7d795642642bde6a262384a0b3: 2024-12-04T20:16:58,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=398cfc7d795642642bde6a262384a0b3, source=c2ef38372881,45313,1733343402534, destination=c2ef38372881,39001,1733343402624, warming up region on c2ef38372881,39001,1733343402624 2024-12-04T20:16:58,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=398cfc7d795642642bde6a262384a0b3, source=c2ef38372881,45313,1733343402534, destination=c2ef38372881,39001,1733343402624, running balancer 2024-12-04T20:16:58,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE 2024-12-04T20:16:58,803 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE 2024-12-04T20:16:58,805 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=CLOSING, regionLocation=c2ef38372881,45313,1733343402534 2024-12-04T20:16:58,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(7855): Warmup {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:58,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:58,808 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,808 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE because future has completed 2024-12-04T20:16:58,809 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:16:58,809 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:58,809 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-04T20:16:58,809 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534}] 2024-12-04T20:16:58,818 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc 2024-12-04T20:16:58,824 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/e13116986a554fda865d8a8a647339fa 2024-12-04T20:16:58,830 INFO [StoreFileOpener-398cfc7d795642642bde6a262384a0b3-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,830 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,830 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:58,831 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,832 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:16:58,832 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:58,840 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/a75d138285a44174bde94b30d82c6843 2024-12-04T20:16:58,845 INFO [StoreFileOpener-398cfc7d795642642bde6a262384a0b3-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,846 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,846 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:58,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:58,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,847 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=39001 {}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343418846Disabling compacts and flushes for region at 1733343418846Disabling writes for close at 1733343418846Writing region close event to WAL at 1733343418847 (+1 ms)Closed at 1733343418847 2024-12-04T20:16:58,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-04T20:16:58,963 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,963 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-04T20:16:58,964 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:58,964 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,964 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,964 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:58,964 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,964 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc, hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b] to archive 2024-12-04T20:16:58,967 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:16:58,971 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/1d30bcdd9ee245ed9368c8f03860cafc 2024-12-04T20:16:58,973 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/f4f44112de3c41a0aec4b6bc926c529b 2024-12-04T20:16:58,985 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f] to archive 2024-12-04T20:16:58,986 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:16:58,988 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/df44c659d0e84dbab5ae9e65e42e4d4f 2024-12-04T20:16:58,993 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-04T20:16:58,994 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:58,994 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343418964Running coprocessor pre-close hooks at 1733343418964Disabling compacts and flushes for region at 1733343418964Disabling writes for close at 1733343418964Writing region close event to WAL at 1733343418990 (+26 ms)Running coprocessor post-close hooks at 1733343418994 (+4 ms)Closed at 1733343418994 2024-12-04T20:16:58,994 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 398cfc7d795642642bde6a262384a0b3 move to c2ef38372881,39001,1733343402624 record at close sequenceid=12 2024-12-04T20:16:58,996 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:58,997 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=CLOSED 2024-12-04T20:16:58,999 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 because future has completed 2024-12-04T20:16:59,004 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-04T20:16:59,005 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 in 192 msec 2024-12-04T20:16:59,006 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE; state=CLOSED, location=c2ef38372881,39001,1733343402624; forceNewPlan=false, retain=false 2024-12-04T20:16:59,156 INFO [c2ef38372881:37947 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-04T20:16:59,157 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPENING, regionLocation=c2ef38372881,39001,1733343402624 2024-12-04T20:16:59,161 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE because future has completed 2024-12-04T20:16:59,161 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624}] 2024-12-04T20:16:59,323 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,324 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:16:59,324 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,324 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:16:59,325 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,325 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,326 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,327 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:16:59,327 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:59,334 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/e13116986a554fda865d8a8a647339fa 2024-12-04T20:16:59,334 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:59,334 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,335 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:16:59,336 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:16:59,342 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/a75d138285a44174bde94b30d82c6843 2024-12-04T20:16:59,342 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:16:59,342 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,343 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,344 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,345 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,345 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,346 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T20:16:59,347 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,348 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 398cfc7d795642642bde6a262384a0b3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67153903, jitterRate=6.711333990097046E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T20:16:59,348 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,348 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 398cfc7d795642642bde6a262384a0b3: Running coprocessor pre-open hook at 1733343419325Writing region info on filesystem at 1733343419325Initializing all the Stores at 1733343419326 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343419326Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343419326Cleaning up temporary data from old regions at 1733343419345 (+19 ms)Running coprocessor post-open hooks at 1733343419348 (+3 ms)Region opened successfully at 1733343419348 2024-12-04T20:16:59,349 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., pid=12, masterSystemTime=1733343419314 2024-12-04T20:16:59,352 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,352 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,353 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPEN, openSeqNum=18, regionLocation=c2ef38372881,39001,1733343402624 2024-12-04T20:16:59,356 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 because future has completed 2024-12-04T20:16:59,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-04T20:16:59,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,39001,1733343402624 in 196 msec 2024-12-04T20:16:59,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, REOPEN/MOVE in 559 msec 2024-12-04T20:16:59,406 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-04T20:16:59,408 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43180, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-04T20:16:59,411 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server c2ef38372881,39001,1733343402624: testing ***** 2024-12-04T20:16:59,411 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-04T20:16:59,414 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-04T20:16:59,416 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-04T20:16:59,420 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-04T20:16:59,421 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-04T20:16:59,427 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 479599408 }, "NonHeapMemoryUsage": { "committed": 171311104, "init": 7667712, "max": -1, "used": 168733800 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "c2ef38372881", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 8, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 2, "ProcessCallTime_median": 4, "ProcessCallTime_75th_percentile": 6, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 7, "ProcessCallTime_98th_percentile": 7, "ProcessCallTime_99th_percentile": 7, "ProcessCallTime_99.9th_percentile": 7, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 0, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 8, "TotalCallTime_mean": 3, "TotalCallTime_25th_percentile": 2, "TotalCallTime_median": 4, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 7, "TotalCallTime_98th_percentile": 7, "TotalCallTime_99th_percentile": 7, "TotalCallTime_99.9th_percentile": 7, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "c2ef38372881", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:55668", "tag.serverName": "c2ef38372881,45313,1733343402534", "tag.clusterId": "6acde3c5-475c-4c4e-93e8-348877078bfd", "tag.Context": "regionserver", "tag.Hostname": "c2ef38372881", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733343402534, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-04T20:16:59,430 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37947 {}] master.MasterRpcServices(700): c2ef38372881,39001,1733343402624 reported a fatal error: ***** ABORTING region server c2ef38372881,39001,1733343402624: testing ***** 2024-12-04T20:16:59,432 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,39001,1733343402624' ***** 2024-12-04T20:16:59,432 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-04T20:16:59,432 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:16:59,432 INFO [RS:1;c2ef38372881:39001 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-04T20:16:59,432 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:16:59,432 INFO [RS:1;c2ef38372881:39001 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-04T20:16:59,433 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(3091): Received CLOSE for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,433 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(956): aborting server c2ef38372881,39001,1733343402624 2024-12-04T20:16:59,433 INFO [RS:1;c2ef38372881:39001 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:16:59,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45313 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:40310 deadline: 1733343479433, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=39001 startCode=1733343402624. As of locationSeqNum=12. 2024-12-04T20:16:59,433 INFO [RS:1;c2ef38372881:39001 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;c2ef38372881:39001. 2024-12-04T20:16:59,433 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:16:59,433 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,434 DEBUG [RS:1;c2ef38372881:39001 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:16:59,434 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,434 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:16:59,434 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,434 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=39001 startCode=1733343402624. As of locationSeqNum=12. 2024-12-04T20:16:59,434 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=39001 startCode=1733343402624. As of locationSeqNum=12. 2024-12-04T20:16:59,434 DEBUG [RPCClient-NioEventLoopGroup-6-7 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=c2ef38372881 port=39001 startCode=1733343402624. As of locationSeqNum=12. 2024-12-04T20:16:59,434 DEBUG [RS:1;c2ef38372881:39001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:16:59,434 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:16:59,434 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1325): Online Regions={398cfc7d795642642bde6a262384a0b3=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.} 2024-12-04T20:16:59,435 DEBUG [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1351): Waiting on 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:16:59,437 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,437 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343419433Running coprocessor pre-close hooks at 1733343419433Disabling compacts and flushes for region at 1733343419433Disabling writes for close at 1733343419434 (+1 ms)Writing region close event to WAL at 1733343419436 (+2 ms)Running coprocessor post-close hooks at 1733343419436Closed at 1733343419437 (+1 ms) 2024-12-04T20:16:59,437 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:16:59,515 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:16:59,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2ef38372881,39001,1733343402624 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:59,544 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2ef38372881,39001,1733343402624 aborting 2024-12-04T20:16:59,544 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server c2ef38372881,39001,1733343402624 aborting 2024-12-04T20:16:59,544 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=12 from cache 2024-12-04T20:16:59,635 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,39001,1733343402624; all regions closed. 2024-12-04T20:16:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741833_1009 (size=2288) 2024-12-04T20:16:59,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741833_1009 (size=2288) 2024-12-04T20:16:59,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741833_1009 (size=2288) 2024-12-04T20:16:59,643 DEBUG [RS:1;c2ef38372881:39001 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:16:59,643 INFO [RS:1;c2ef38372881:39001 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:16:59,643 INFO [RS:1;c2ef38372881:39001 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:16:59,644 INFO [RS:1;c2ef38372881:39001 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-04T20:16:59,644 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:16:59,644 INFO [RS:1;c2ef38372881:39001 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:16:59,644 INFO [RS:1;c2ef38372881:39001 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:16:59,645 INFO [RS:1;c2ef38372881:39001 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:16:59,645 INFO [RS:1;c2ef38372881:39001 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:16:59,645 INFO [RS:1;c2ef38372881:39001 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:39001 2024-12-04T20:16:59,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,39001,1733343402624 2024-12-04T20:16:59,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:16:59,651 INFO [RS:1;c2ef38372881:39001 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:16:59,652 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,39001,1733343402624] 2024-12-04T20:16:59,653 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,39001,1733343402624 already deleted, retry=false 2024-12-04T20:16:59,653 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of c2ef38372881,39001,1733343402624 on c2ef38372881,37947,1733343401962 2024-12-04T20:16:59,658 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure c2ef38372881,39001,1733343402624, splitWal=true, meta=false 2024-12-04T20:16:59,660 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for c2ef38372881,39001,1733343402624 (carryingMeta=false) c2ef38372881,39001,1733343402624/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@679d8bb4[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-04T20:16:59,661 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure c2ef38372881,39001,1733343402624, splitWal=true, meta=false 2024-12-04T20:16:59,662 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): c2ef38372881,39001,1733343402624 had 1 regions 2024-12-04T20:16:59,663 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure c2ef38372881,39001,1733343402624, splitWal=true, meta=false, isMeta: false 2024-12-04T20:16:59,664 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting 2024-12-04T20:16:59,665 INFO [PEWorker-1 {}] master.SplitWALManager(105): c2ef38372881,39001,1733343402624 WAL count=1, meta=false 2024-12-04T20:16:59,667 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure c2ef38372881%2C39001%2C1733343402624.1733343403847}] 2024-12-04T20:16:59,672 DEBUG [PEWorker-2 {}] master.SplitWALManager(158): Acquired split WAL worker=c2ef38372881,45313,1733343402534 2024-12-04T20:16:59,674 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure c2ef38372881%2C39001%2C1733343402624.1733343403847, worker=c2ef38372881,45313,1733343402534}] 2024-12-04T20:16:59,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:16:59,754 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:39001-0x100a6e0b14d0002, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:16:59,755 INFO [RS:1;c2ef38372881:39001 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:16:59,755 INFO [RS:1;c2ef38372881:39001 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,39001,1733343402624; zookeeper connection closed. 2024-12-04T20:16:59,756 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@33686110 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@33686110 2024-12-04T20:16:59,759 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18] 2024-12-04T20:16:59,761 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server c2ef38372881:39001 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2ef38372881/172.17.0.2:39001 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:16:59,762 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18, error=java.net.ConnectException: Call to address=c2ef38372881:39001 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2ef38372881/172.17.0.2:39001 2024-12-04T20:16:59,762 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-04T20:16:59,762 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 from cache 2024-12-04T20:16:59,763 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address c2ef38372881:39001 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: c2ef38372881/172.17.0.2:39001 2024-12-04T20:16:59,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45313 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-04T20:16:59,858 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847, size=2.2 K (2288bytes) 2024-12-04T20:16:59,858 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 2024-12-04T20:16:59,858 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 after 0ms 2024-12-04T20:16:59,881 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:16:59,881 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 took 24ms 2024-12-04T20:16:59,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 398cfc7d795642642bde6a262384a0b3: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-04T20:16:59,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 so closing down 2024-12-04T20:16:59,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:16:59,888 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:16:59,888 INFO [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-04T20:16:59,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847, journal: Splitting hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847, size=2.2 K (2288bytes) at 1733343419858Finishing writing output for hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 so closing down at 1733343419888 (+30 ms)3 split writer threads finished at 1733343419888Processed 6 edits across 0 Regions in 7 ms; skipped=6; WAL=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733343419888 2024-12-04T20:16:59,888 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 2024-12-04T20:16:59,889 DEBUG [RS_LOG_REPLAY_OPS-regionserver/c2ef38372881:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-04T20:16:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37947 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-04T20:16:59,897 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting/c2ef38372881%2C39001%2C1733343402624.1733343403847 to hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs 2024-12-04T20:16:59,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-04T20:16:59,900 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure c2ef38372881%2C39001%2C1733343402624.1733343403847, worker=c2ef38372881,45313,1733343402534 in 223 msec 2024-12-04T20:16:59,901 DEBUG [PEWorker-4 {}] master.SplitWALManager(172): Release split WAL worker=c2ef38372881,45313,1733343402534 2024-12-04T20:16:59,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-04T20:16:59,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure c2ef38372881%2C39001%2C1733343402624.1733343403847, worker=c2ef38372881,45313,1733343402534 in 234 msec 2024-12-04T20:16:59,905 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting dir is empty, no logs to split. 2024-12-04T20:16:59,905 INFO [PEWorker-1 {}] master.SplitWALManager(105): c2ef38372881,39001,1733343402624 WAL count=0, meta=false 2024-12-04T20:16:59,905 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if c2ef38372881,39001,1733343402624 WAL splitting is done? wals=0, meta=false 2024-12-04T20:16:59,907 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for c2ef38372881,39001,1733343402624 failed, ignore...File hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/WALs/c2ef38372881,39001,1733343402624-splitting does not exist. 2024-12-04T20:16:59,909 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN}] 2024-12-04T20:16:59,911 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN 2024-12-04T20:16:59,912 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-04T20:17:00,063 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(204): Hosts are {c2ef38372881=0} racks are {/default-rack=0} 2024-12-04T20:17:00,063 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-04T20:17:00,063 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-04T20:17:00,063 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-04T20:17:00,063 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-04T20:17:00,064 INFO [c2ef38372881:37947 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-04T20:17:00,064 INFO [c2ef38372881:37947 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-04T20:17:00,064 DEBUG [c2ef38372881:37947 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-04T20:17:00,065 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPENING, regionLocation=c2ef38372881,45313,1733343402534 2024-12-04T20:17:00,071 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN because future has completed 2024-12-04T20:17:00,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534}] 2024-12-04T20:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18] 2024-12-04T20:17:00,075 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to c2ef38372881:39001 this server is in the failed servers list 2024-12-04T20:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=c2ef38372881:39001 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c2ef38372881:39001 2024-12-04T20:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: c2ef38372881:39001 2024-12-04T20:17:00,076 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,39001,1733343402624, seqNum=18 from cache 2024-12-04T20:17:00,229 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:00,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 398cfc7d795642642bde6a262384a0b3, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:00,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:00,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,230 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,233 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,234 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf1 2024-12-04T20:17:00,234 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,243 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf1/e13116986a554fda865d8a8a647339fa 2024-12-04T20:17:00,243 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,243 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,244 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 398cfc7d795642642bde6a262384a0b3 columnFamilyName cf2 2024-12-04T20:17:00,244 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,250 DEBUG [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/cf2/a75d138285a44174bde94b30d82c6843 2024-12-04T20:17:00,250 INFO [StoreOpener-398cfc7d795642642bde6a262384a0b3-1 {}] regionserver.HStore(327): Store=398cfc7d795642642bde6a262384a0b3/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,250 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,251 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,252 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,253 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,253 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,254 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-04T20:17:00,255 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,256 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 398cfc7d795642642bde6a262384a0b3; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62170449, jitterRate=-0.07358811795711517}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-04T20:17:00,256 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:00,257 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 398cfc7d795642642bde6a262384a0b3: Running coprocessor pre-open hook at 1733343420230Writing region info on filesystem at 1733343420230Initializing all the Stores at 1733343420232 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420232Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420232Cleaning up temporary data from old regions at 1733343420253 (+21 ms)Running coprocessor post-open hooks at 1733343420256 (+3 ms)Region opened successfully at 1733343420257 (+1 ms) 2024-12-04T20:17:00,258 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., pid=17, masterSystemTime=1733343420224 2024-12-04T20:17:00,260 DEBUG [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:00,260 INFO [RS_OPEN_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:00,260 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=398cfc7d795642642bde6a262384a0b3, regionState=OPEN, openSeqNum=18, regionLocation=c2ef38372881,45313,1733343402534 2024-12-04T20:17:00,262 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 because future has completed 2024-12-04T20:17:00,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-04T20:17:00,265 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 398cfc7d795642642bde6a262384a0b3, server=c2ef38372881,45313,1733343402534 in 191 msec 2024-12-04T20:17:00,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-04T20:17:00,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=398cfc7d795642642bde6a262384a0b3, ASSIGN in 356 msec 2024-12-04T20:17:00,267 INFO [PEWorker-4 {}] procedure.ServerCrashProcedure(291): removed crashed server c2ef38372881,39001,1733343402624 after splitting done 2024-12-04T20:17:00,269 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure c2ef38372881,39001,1733343402624, splitWal=true, meta=false in 613 msec 2024-12-04T20:17:00,588 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3., hostname=c2ef38372881,45313,1733343402534, seqNum=18] 2024-12-04T20:17:00,605 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=406 (was 408), OpenFileDescriptor=1048 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=166 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=4984 (was 5023) 2024-12-04T20:17:00,607 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-04T20:17:00,618 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=406, OpenFileDescriptor=1048, MaxFileDescriptor=1048576, SystemLoadAverage=166, ProcessCount=11, AvailableMemoryMB=4983 2024-12-04T20:17:00,619 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1048 is superior to 1024 2024-12-04T20:17:00,635 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:00,636 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:00,637 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:17:00,639 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-32041909, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-32041909, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:00,650 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-32041909/hregion-32041909.1733343420640, exclude list is [], retry=0 2024-12-04T20:17:00,653 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:00,653 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:00,653 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:00,675 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-32041909/hregion-32041909.1733343420640 2024-12-04T20:17:00,675 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:00,675 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 83e54f9a4c804ba53063040367b95cbe, NAME => 'testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:17:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741893_1071 (size=67) 2024-12-04T20:17:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741893_1071 (size=67) 2024-12-04T20:17:00,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741893_1071 (size=67) 2024-12-04T20:17:00,683 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:00,684 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,685 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName a 2024-12-04T20:17:00,686 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,686 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,686 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,687 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName b 2024-12-04T20:17:00,687 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,688 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,688 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,689 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName c 2024-12-04T20:17:00,689 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,690 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,690 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,691 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,692 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,692 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,693 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:00,694 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,697 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:17:00,697 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 83e54f9a4c804ba53063040367b95cbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73244045, jitterRate=0.09142132103443146}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 83e54f9a4c804ba53063040367b95cbe: Writing region info on filesystem at 1733343420683Initializing all the Stores at 1733343420684 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420684Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420684Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420684Cleaning up temporary data from old regions at 1733343420692 (+8 ms)Region opened successfully at 1733343420698 (+6 ms) 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 83e54f9a4c804ba53063040367b95cbe, disabling compactions & flushes 2024-12-04T20:17:00,698 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. after waiting 0 ms 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,698 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,698 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 83e54f9a4c804ba53063040367b95cbe: Waiting for close lock at 1733343420698Disabling compacts and flushes for region at 1733343420698Disabling writes for close at 1733343420698Writing region close event to WAL at 1733343420698Closed at 1733343420698 2024-12-04T20:17:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741892_1070 (size=175) 2024-12-04T20:17:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741892_1070 (size=175) 2024-12-04T20:17:00,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741892_1070 (size=175) 2024-12-04T20:17:00,704 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:00,704 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-32041909:(num 1733343420640) 2024-12-04T20:17:00,704 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:00,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:00,717 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, exclude list is [], retry=0 2024-12-04T20:17:00,719 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:00,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:00,720 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:00,743 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 2024-12-04T20:17:00,743 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:00,743 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 83e54f9a4c804ba53063040367b95cbe, NAME => 'testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:00,743 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:00,744 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,744 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,745 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,746 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName a 2024-12-04T20:17:00,746 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,746 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,747 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,747 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName b 2024-12-04T20:17:00,747 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,748 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,748 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,749 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName c 2024-12-04T20:17:00,749 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:00,749 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:00,749 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,750 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,751 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,752 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,752 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,753 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:00,754 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:00,755 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 83e54f9a4c804ba53063040367b95cbe; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69759405, jitterRate=0.03949613869190216}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:00,755 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 83e54f9a4c804ba53063040367b95cbe: Writing region info on filesystem at 1733343420744Initializing all the Stores at 1733343420745 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420745Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420745Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343420745Cleaning up temporary data from old regions at 1733343420752 (+7 ms)Region opened successfully at 1733343420755 (+3 ms) 2024-12-04T20:17:00,779 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83e54f9a4c804ba53063040367b95cbe 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T20:17:00,794 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/a/397d89b000344abf87486453d6bc8c37 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733343420755/Put/seqid=0 2024-12-04T20:17:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741895_1073 (size=5958) 2024-12-04T20:17:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741895_1073 (size=5958) 2024-12-04T20:17:00,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741895_1073 (size=5958) 2024-12-04T20:17:00,802 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/a/397d89b000344abf87486453d6bc8c37 2024-12-04T20:17:00,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/65cc8ce7f0f9427199f2e9092721eae7 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733343420763/Put/seqid=0 2024-12-04T20:17:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741896_1074 (size=5958) 2024-12-04T20:17:00,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741896_1074 (size=5958) 2024-12-04T20:17:00,835 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741896_1074 (size=5958) 2024-12-04T20:17:00,835 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/65cc8ce7f0f9427199f2e9092721eae7 2024-12-04T20:17:00,861 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/c/c68566e2905647609f55276ce1fdaf35 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733343420770/Put/seqid=0 2024-12-04T20:17:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741897_1075 (size=5958) 2024-12-04T20:17:00,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741897_1075 (size=5958) 2024-12-04T20:17:00,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741897_1075 (size=5958) 2024-12-04T20:17:00,870 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/c/c68566e2905647609f55276ce1fdaf35 2024-12-04T20:17:00,876 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/a/397d89b000344abf87486453d6bc8c37 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/a/397d89b000344abf87486453d6bc8c37 2024-12-04T20:17:00,882 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/a/397d89b000344abf87486453d6bc8c37, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T20:17:00,884 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/65cc8ce7f0f9427199f2e9092721eae7 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/65cc8ce7f0f9427199f2e9092721eae7 2024-12-04T20:17:00,889 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/65cc8ce7f0f9427199f2e9092721eae7, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T20:17:00,891 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/c/c68566e2905647609f55276ce1fdaf35 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/c/c68566e2905647609f55276ce1fdaf35 2024-12-04T20:17:00,897 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/c/c68566e2905647609f55276ce1fdaf35, entries=10, sequenceid=33, filesize=5.8 K 2024-12-04T20:17:00,898 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 83e54f9a4c804ba53063040367b95cbe in 119ms, sequenceid=33, compaction requested=false 2024-12-04T20:17:00,898 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 83e54f9a4c804ba53063040367b95cbe: 2024-12-04T20:17:00,898 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 83e54f9a4c804ba53063040367b95cbe, disabling compactions & flushes 2024-12-04T20:17:00,898 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,899 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,899 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. after waiting 0 ms 2024-12-04T20:17:00,899 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,900 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. 2024-12-04T20:17:00,900 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 83e54f9a4c804ba53063040367b95cbe: Waiting for close lock at 1733343420898Disabling compacts and flushes for region at 1733343420898Disabling writes for close at 1733343420899 (+1 ms)Writing region close event to WAL at 1733343420900 (+1 ms)Closed at 1733343420900 2024-12-04T20:17:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741894_1072 (size=6311) 2024-12-04T20:17:00,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741894_1072 (size=6311) 2024-12-04T20:17:00,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741894_1072 (size=6311) 2024-12-04T20:17:00,909 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/65cc8ce7f0f9427199f2e9092721eae7 to hdfs://localhost:44433/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/65cc8ce7f0f9427199f2e9092721eae7 2024-12-04T20:17:00,930 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, size=6.2 K (6311bytes) 2024-12-04T20:17:00,930 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 2024-12-04T20:17:00,931 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 after 1ms 2024-12-04T20:17:00,954 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:00,954 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 took 24ms 2024-12-04T20:17:00,956 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 so closing down 2024-12-04T20:17:00,956 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:00,957 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733343420706.temp 2024-12-04T20:17:00,980 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp 2024-12-04T20:17:00,980 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741898_1076 (size=5575) 2024-12-04T20:17:00,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741898_1076 (size=5575) 2024-12-04T20:17:00,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741898_1076 (size=5575) 2024-12-04T20:17:00,987 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:00,988 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 2024-12-04T20:17:00,988 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-04T20:17:00,988 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, journal: Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, size=6.2 K (6311bytes) at 1733343420930Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 so closing down at 1733343420956 (+26 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp at 1733343420980 (+24 ms)3 split writer threads finished at 1733343420980Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733343420987 (+7 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000003-wal.1733343420706.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 at 1733343420988 (+1 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733343420988 2024-12-04T20:17:00,990 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420706 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343420706 2024-12-04T20:17:00,991 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 2024-12-04T20:17:00,991 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:00,992 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:01,003 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420992, exclude list is [], retry=0 2024-12-04T20:17:01,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:01,006 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:01,007 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:01,029 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420992 2024-12-04T20:17:01,030 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 83e54f9a4c804ba53063040367b95cbe, NAME => 'testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,030 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,032 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,032 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName a 2024-12-04T20:17:01,032 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,038 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/a/397d89b000344abf87486453d6bc8c37 2024-12-04T20:17:01,038 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,038 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,039 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName b 2024-12-04T20:17:01,039 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,040 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,040 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,041 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 83e54f9a4c804ba53063040367b95cbe columnFamilyName c 2024-12-04T20:17:01,041 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,046 DEBUG [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/c/c68566e2905647609f55276ce1fdaf35 2024-12-04T20:17:01,047 INFO [StoreOpener-83e54f9a4c804ba53063040367b95cbe-1 {}] regionserver.HStore(327): Store=83e54f9a4c804ba53063040367b95cbe/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,047 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,047 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,049 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,050 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 2024-12-04T20:17:01,079 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:01,080 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 2024-12-04T20:17:01,080 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 83e54f9a4c804ba53063040367b95cbe 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T20:17:01,094 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/105ad9fe5be7452fb70916698522fd9f is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733343420763/Put/seqid=0 2024-12-04T20:17:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741900_1078 (size=5958) 2024-12-04T20:17:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741900_1078 (size=5958) 2024-12-04T20:17:01,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741900_1078 (size=5958) 2024-12-04T20:17:01,102 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/105ad9fe5be7452fb70916698522fd9f 2024-12-04T20:17:01,108 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/.tmp/b/105ad9fe5be7452fb70916698522fd9f as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/105ad9fe5be7452fb70916698522fd9f 2024-12-04T20:17:01,114 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/b/105ad9fe5be7452fb70916698522fd9f, entries=10, sequenceid=32, filesize=5.8 K 2024-12-04T20:17:01,114 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 83e54f9a4c804ba53063040367b95cbe in 34ms, sequenceid=32, compaction requested=false; wal=null 2024-12-04T20:17:01,115 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/0000000000000000032 2024-12-04T20:17:01,116 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,116 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,116 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:01,118 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 83e54f9a4c804ba53063040367b95cbe 2024-12-04T20:17:01,120 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/83e54f9a4c804ba53063040367b95cbe/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-04T20:17:01,120 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 83e54f9a4c804ba53063040367b95cbe; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68734988, jitterRate=0.024231135845184326}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:01,121 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 83e54f9a4c804ba53063040367b95cbe: Writing region info on filesystem at 1733343421030Initializing all the Stores at 1733343421031 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421031Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421031Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421031Obtaining lock to block concurrent updates at 1733343421080 (+49 ms)Preparing flush snapshotting stores in 83e54f9a4c804ba53063040367b95cbe at 1733343421080Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733343421081 (+1 ms)Flushing stores of testReplayEditsWrittenViaHRegion,,1733343420635.83e54f9a4c804ba53063040367b95cbe. at 1733343421081Flushing 83e54f9a4c804ba53063040367b95cbe/b: creating writer at 1733343421081Flushing 83e54f9a4c804ba53063040367b95cbe/b: appending metadata at 1733343421094 (+13 ms)Flushing 83e54f9a4c804ba53063040367b95cbe/b: closing flushed file at 1733343421094Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7a8840f: reopening flushed file at 1733343421107 (+13 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 83e54f9a4c804ba53063040367b95cbe in 34ms, sequenceid=32, compaction requested=false; wal=null at 1733343421114 (+7 ms)Cleaning up temporary data from old regions at 1733343421116 (+2 ms)Region opened successfully at 1733343421121 (+5 ms) 2024-12-04T20:17:01,140 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=415 (was 406) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:34118 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:54298 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:34292 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:47018 [Waiting for operation #52] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:41806 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:34158 [Waiting for operation #25] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1124 (was 1048) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=161 (was 166), ProcessCount=11 (was 11), AvailableMemoryMB=4975 (was 4983) 2024-12-04T20:17:01,140 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-04T20:17:01,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415, OpenFileDescriptor=1124, MaxFileDescriptor=1048576, SystemLoadAverage=161, ProcessCount=11, AvailableMemoryMB=4974 2024-12-04T20:17:01,151 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1124 is superior to 1024 2024-12-04T20:17:01,163 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:01,165 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:01,166 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:17:01,169 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-98686075, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-98686075, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:01,180 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-98686075/hregion-98686075.1733343421169, exclude list is [], retry=0 2024-12-04T20:17:01,183 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:01,183 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:01,184 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:01,207 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-98686075/hregion-98686075.1733343421169 2024-12-04T20:17:01,208 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:17:01,208 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 211bd85f25734d69014eefe731f00619, NAME => 'testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:17:01,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741902_1080 (size=68) 2024-12-04T20:17:01,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741902_1080 (size=68) 2024-12-04T20:17:01,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741902_1080 (size=68) 2024-12-04T20:17:01,219 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:01,220 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,221 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName a 2024-12-04T20:17:01,221 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,222 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,222 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,223 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName b 2024-12-04T20:17:01,223 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,224 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,224 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,225 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName c 2024-12-04T20:17:01,225 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,226 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,226 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,226 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,227 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,228 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,228 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,228 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:01,229 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,232 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:17:01,232 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 211bd85f25734d69014eefe731f00619; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72656133, jitterRate=0.0826607495546341}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 211bd85f25734d69014eefe731f00619: Writing region info on filesystem at 1733343421219Initializing all the Stores at 1733343421219Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421219Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421220 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421220Cleaning up temporary data from old regions at 1733343421228 (+8 ms)Region opened successfully at 1733343421233 (+5 ms) 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 211bd85f25734d69014eefe731f00619, disabling compactions & flushes 2024-12-04T20:17:01,233 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. after waiting 0 ms 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:01,233 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:01,233 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 211bd85f25734d69014eefe731f00619: Waiting for close lock at 1733343421233Disabling compacts and flushes for region at 1733343421233Disabling writes for close at 1733343421233Writing region close event to WAL at 1733343421233Closed at 1733343421233 2024-12-04T20:17:01,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741901_1079 (size=175) 2024-12-04T20:17:01,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741901_1079 (size=175) 2024-12-04T20:17:01,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741901_1079 (size=175) 2024-12-04T20:17:01,238 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:01,238 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-98686075:(num 1733343421169) 2024-12-04T20:17:01,238 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:01,240 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:01,251 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, exclude list is [], retry=0 2024-12-04T20:17:01,253 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:01,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:01,254 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:01,278 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 2024-12-04T20:17:01,279 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:01,347 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 211bd85f25734d69014eefe731f00619, NAME => 'testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:01,350 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,350 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:01,350 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,350 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,353 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,354 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName a 2024-12-04T20:17:01,354 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,355 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,355 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,356 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName b 2024-12-04T20:17:01,356 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,357 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,357 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,358 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName c 2024-12-04T20:17:01,358 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:01,359 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:01,359 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,360 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,361 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,361 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,361 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,362 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:01,363 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,363 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 211bd85f25734d69014eefe731f00619; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62521605, jitterRate=-0.0683554857969284}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:01,363 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:01,364 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 211bd85f25734d69014eefe731f00619: Running coprocessor pre-open hook at 1733343421351Writing region info on filesystem at 1733343421351Initializing all the Stores at 1733343421352 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421352Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421352Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343421352Cleaning up temporary data from old regions at 1733343421361 (+9 ms)Running coprocessor post-open hooks at 1733343421363 (+2 ms)Region opened successfully at 1733343421364 (+1 ms) 2024-12-04T20:17:01,377 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 211bd85f25734d69014eefe731f00619 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-04T20:17:01,378 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:02,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-04T20:17:02,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:02,303 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-04T20:17:02,303 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:02,305 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-04T20:17:02,305 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:02,307 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-04T20:17:02,307 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:02,308 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-04T20:17:02,308 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:02,379 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:03,119 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:17:03,379 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:04,381 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:05,382 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:06,383 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:07,384 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:08,386 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:09,387 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:10,388 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 211bd85f25734d69014eefe731f00619/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:10,391 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 211bd85f25734d69014eefe731f00619: 2024-12-04T20:17:10,391 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 211bd85f25734d69014eefe731f00619: 2024-12-04T20:17:10,409 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-04T20:17:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 211bd85f25734d69014eefe731f00619, disabling compactions & flushes 2024-12-04T20:17:10,409 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. after waiting 0 ms 2024-12-04T20:17:10,409 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,409 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,409 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. 2024-12-04T20:17:10,410 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 211bd85f25734d69014eefe731f00619: Waiting for close lock at 1733343430409Running coprocessor pre-close hooks at 1733343430409Disabling compacts and flushes for region at 1733343430409Disabling writes for close at 1733343430409Writing region close event to WAL at 1733343430409Running coprocessor post-close hooks at 1733343430409Closed at 1733343430409 2024-12-04T20:17:10,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741903_1081 (size=4236) 2024-12-04T20:17:10,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741903_1081 (size=4236) 2024-12-04T20:17:10,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741903_1081 (size=4236) 2024-12-04T20:17:10,425 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, size=4.1 K (4236bytes) 2024-12-04T20:17:10,425 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 2024-12-04T20:17:10,425 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 after 0ms 2024-12-04T20:17:10,447 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:10,447 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 took 23ms 2024-12-04T20:17:10,449 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 so closing down 2024-12-04T20:17:10,450 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:10,450 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733343421240.temp 2024-12-04T20:17:10,474 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp 2024-12-04T20:17:10,475 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741904_1082 (size=3245) 2024-12-04T20:17:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741904_1082 (size=3245) 2024-12-04T20:17:10,481 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741904_1082 (size=3245) 2024-12-04T20:17:10,481 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:10,483 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 2024-12-04T20:17:10,483 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-04T20:17:10,483 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, journal: Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, size=4.1 K (4236bytes) at 1733343430425Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 so closing down at 1733343430449 (+24 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp at 1733343430474 (+25 ms)3 split writer threads finished at 1733343430475 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733343430481 (+6 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000004-wal.1733343421240.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 at 1733343430483 (+2 ms)Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733343430483 2024-12-04T20:17:10,484 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343421240 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343421240 2024-12-04T20:17:10,485 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 2024-12-04T20:17:10,485 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:10,487 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:10,500 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343430487, exclude list is [], retry=0 2024-12-04T20:17:10,503 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:10,503 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:10,503 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:10,524 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343430487 2024-12-04T20:17:10,525 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:10,525 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 211bd85f25734d69014eefe731f00619, NAME => 'testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:10,526 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,526 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:10,526 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,526 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,528 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,528 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName a 2024-12-04T20:17:10,529 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:10,529 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:10,529 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,530 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName b 2024-12-04T20:17:10,530 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:10,530 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:10,531 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,531 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 211bd85f25734d69014eefe731f00619 columnFamilyName c 2024-12-04T20:17:10,531 DEBUG [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:10,532 INFO [StoreOpener-211bd85f25734d69014eefe731f00619-1 {}] regionserver.HStore(327): Store=211bd85f25734d69014eefe731f00619/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:10,532 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,533 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,535 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,536 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 2024-12-04T20:17:10,570 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:10,571 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 2024-12-04T20:17:10,571 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 211bd85f25734d69014eefe731f00619 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-04T20:17:10,584 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/a/6e0f4f1028284684bd2aa003d6e6299f is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733343430402/Put/seqid=0 2024-12-04T20:17:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741906_1084 (size=5523) 2024-12-04T20:17:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741906_1084 (size=5523) 2024-12-04T20:17:10,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741906_1084 (size=5523) 2024-12-04T20:17:10,591 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/a/6e0f4f1028284684bd2aa003d6e6299f 2024-12-04T20:17:10,616 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/b/76cfea9fd4c24ec7bd4aa85aab1351c0 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733343430393/Put/seqid=0 2024-12-04T20:17:10,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741907_1085 (size=5524) 2024-12-04T20:17:10,623 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/b/76cfea9fd4c24ec7bd4aa85aab1351c0 2024-12-04T20:17:10,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741907_1085 (size=5524) 2024-12-04T20:17:10,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741907_1085 (size=5524) 2024-12-04T20:17:10,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/c/4af7bd1dc3ce4c0fa7904a207e8769c4 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733343430399/Put/seqid=0 2024-12-04T20:17:10,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741908_1086 (size=5457) 2024-12-04T20:17:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741908_1086 (size=5457) 2024-12-04T20:17:10,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741908_1086 (size=5457) 2024-12-04T20:17:10,652 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/c/4af7bd1dc3ce4c0fa7904a207e8769c4 2024-12-04T20:17:10,660 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/a/6e0f4f1028284684bd2aa003d6e6299f as hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/a/6e0f4f1028284684bd2aa003d6e6299f 2024-12-04T20:17:10,667 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/a/6e0f4f1028284684bd2aa003d6e6299f, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T20:17:10,668 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/b/76cfea9fd4c24ec7bd4aa85aab1351c0 as hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/b/76cfea9fd4c24ec7bd4aa85aab1351c0 2024-12-04T20:17:10,676 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/b/76cfea9fd4c24ec7bd4aa85aab1351c0, entries=7, sequenceid=26, filesize=5.4 K 2024-12-04T20:17:10,677 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/.tmp/c/4af7bd1dc3ce4c0fa7904a207e8769c4 as hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/c/4af7bd1dc3ce4c0fa7904a207e8769c4 2024-12-04T20:17:10,684 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/c/4af7bd1dc3ce4c0fa7904a207e8769c4, entries=6, sequenceid=26, filesize=5.3 K 2024-12-04T20:17:10,684 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 211bd85f25734d69014eefe731f00619 in 113ms, sequenceid=26, compaction requested=false; wal=null 2024-12-04T20:17:10,685 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/0000000000000000026 2024-12-04T20:17:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,687 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,688 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:10,690 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,692 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsAfterAbortingFlush/211bd85f25734d69014eefe731f00619/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-04T20:17:10,694 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 211bd85f25734d69014eefe731f00619; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65305264, jitterRate=-0.026875734329223633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:10,694 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 211bd85f25734d69014eefe731f00619 2024-12-04T20:17:10,694 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 211bd85f25734d69014eefe731f00619: Running coprocessor pre-open hook at 1733343430526Writing region info on filesystem at 1733343430526Initializing all the Stores at 1733343430527 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343430527Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343430527Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343430527Obtaining lock to block concurrent updates at 1733343430571 (+44 ms)Preparing flush snapshotting stores in 211bd85f25734d69014eefe731f00619 at 1733343430571Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733343430572 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733343421164.211bd85f25734d69014eefe731f00619. at 1733343430572Flushing 211bd85f25734d69014eefe731f00619/a: creating writer at 1733343430572Flushing 211bd85f25734d69014eefe731f00619/a: appending metadata at 1733343430584 (+12 ms)Flushing 211bd85f25734d69014eefe731f00619/a: closing flushed file at 1733343430584Flushing 211bd85f25734d69014eefe731f00619/b: creating writer at 1733343430596 (+12 ms)Flushing 211bd85f25734d69014eefe731f00619/b: appending metadata at 1733343430615 (+19 ms)Flushing 211bd85f25734d69014eefe731f00619/b: closing flushed file at 1733343430615Flushing 211bd85f25734d69014eefe731f00619/c: creating writer at 1733343430629 (+14 ms)Flushing 211bd85f25734d69014eefe731f00619/c: appending metadata at 1733343430642 (+13 ms)Flushing 211bd85f25734d69014eefe731f00619/c: closing flushed file at 1733343430642Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@23cd9dc3: reopening flushed file at 1733343430659 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@326a40c8: reopening flushed file at 1733343430667 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ede033c: reopening flushed file at 1733343430676 (+9 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 211bd85f25734d69014eefe731f00619 in 113ms, sequenceid=26, compaction requested=false; wal=null at 1733343430684 (+8 ms)Cleaning up temporary data from old regions at 1733343430687 (+3 ms)Running coprocessor post-open hooks at 1733343430694 (+7 ms)Region opened successfully at 1733343430694 2024-12-04T20:17:10,716 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=415 (was 415), OpenFileDescriptor=1182 (was 1124) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=148 (was 161), ProcessCount=11 (was 11), AvailableMemoryMB=4947 (was 4974) 2024-12-04T20:17:10,716 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-04T20:17:10,729 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=415, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=148, ProcessCount=11, AvailableMemoryMB=4946 2024-12-04T20:17:10,729 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-04T20:17:10,748 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:10,750 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:10,751 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:17:10,754 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-39524658, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-39524658, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:10,766 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-39524658/hregion-39524658.1733343430754, exclude list is [], retry=0 2024-12-04T20:17:10,769 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:10,770 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:10,770 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:10,795 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-39524658/hregion-39524658.1733343430754 2024-12-04T20:17:10,795 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:10,795 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dce577b5f5a6f191c56d001f4e870372, NAME => 'testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:17:10,804 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741910_1088 (size=61) 2024-12-04T20:17:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741910_1088 (size=61) 2024-12-04T20:17:10,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741910_1088 (size=61) 2024-12-04T20:17:10,806 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:10,807 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,808 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce577b5f5a6f191c56d001f4e870372 columnFamilyName a 2024-12-04T20:17:10,808 DEBUG [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:10,809 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(327): Store=dce577b5f5a6f191c56d001f4e870372/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:10,809 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,811 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,812 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,814 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,816 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:17:10,817 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dce577b5f5a6f191c56d001f4e870372; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62405683, jitterRate=-0.07008285820484161}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:17:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dce577b5f5a6f191c56d001f4e870372: Writing region info on filesystem at 1733343430806Initializing all the Stores at 1733343430806Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343430807 (+1 ms)Cleaning up temporary data from old regions at 1733343430812 (+5 ms)Region opened successfully at 1733343430817 (+5 ms) 2024-12-04T20:17:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dce577b5f5a6f191c56d001f4e870372, disabling compactions & flushes 2024-12-04T20:17:10,817 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. after waiting 0 ms 2024-12-04T20:17:10,817 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,818 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,818 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dce577b5f5a6f191c56d001f4e870372: Waiting for close lock at 1733343430817Disabling compacts and flushes for region at 1733343430817Disabling writes for close at 1733343430817Writing region close event to WAL at 1733343430818 (+1 ms)Closed at 1733343430818 2024-12-04T20:17:10,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741909_1087 (size=175) 2024-12-04T20:17:10,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741909_1087 (size=175) 2024-12-04T20:17:10,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741909_1087 (size=175) 2024-12-04T20:17:10,824 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:10,824 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-39524658:(num 1733343430754) 2024-12-04T20:17:10,824 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:10,826 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:10,838 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, exclude list is [], retry=0 2024-12-04T20:17:10,841 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:10,841 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:10,842 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:10,866 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 2024-12-04T20:17:10,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:17:10,866 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dce577b5f5a6f191c56d001f4e870372, NAME => 'testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:10,867 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:10,867 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,867 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,868 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,869 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce577b5f5a6f191c56d001f4e870372 columnFamilyName a 2024-12-04T20:17:10,869 DEBUG [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:10,870 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(327): Store=dce577b5f5a6f191c56d001f4e870372/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:10,870 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,871 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,872 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,873 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,875 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:10,876 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dce577b5f5a6f191c56d001f4e870372; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74176438, jitterRate=0.10531505942344666}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:17:10,876 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dce577b5f5a6f191c56d001f4e870372: Writing region info on filesystem at 1733343430867Initializing all the Stores at 1733343430868 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343430868Cleaning up temporary data from old regions at 1733343430873 (+5 ms)Region opened successfully at 1733343430876 (+3 ms) 2024-12-04T20:17:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dce577b5f5a6f191c56d001f4e870372, disabling compactions & flushes 2024-12-04T20:17:10,883 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,883 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,884 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. after waiting 0 ms 2024-12-04T20:17:10,884 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,884 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,884 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. 2024-12-04T20:17:10,884 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dce577b5f5a6f191c56d001f4e870372: Waiting for close lock at 1733343430883Disabling compacts and flushes for region at 1733343430883Disabling writes for close at 1733343430884 (+1 ms)Writing region close event to WAL at 1733343430884Closed at 1733343430884 2024-12-04T20:17:10,887 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 not finished, retry = 0 2024-12-04T20:17:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741911_1089 (size=1795) 2024-12-04T20:17:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741911_1089 (size=1795) 2024-12-04T20:17:10,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741911_1089 (size=1795) 2024-12-04T20:17:11,001 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, size=1.8 K (1795bytes) 2024-12-04T20:17:11,001 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 2024-12-04T20:17:11,002 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 after 1ms 2024-12-04T20:17:11,026 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:11,026 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 took 25ms 2024-12-04T20:17:11,028 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 so closing down 2024-12-04T20:17:11,028 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:11,029 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733343430826.temp 2024-12-04T20:17:11,051 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp 2024-12-04T20:17:11,052 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741912_1090 (size=1795) 2024-12-04T20:17:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741912_1090 (size=1795) 2024-12-04T20:17:11,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741912_1090 (size=1795) 2024-12-04T20:17:11,059 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:11,060 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp to hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 2024-12-04T20:17:11,060 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-04T20:17:11,060 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, journal: Splitting hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, size=1.8 K (1795bytes) at 1733343431001Finishing writing output for hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 so closing down at 1733343431028 (+27 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp at 1733343431051 (+23 ms)3 split writer threads finished at 1733343431052 (+1 ms)Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733343431059 (+7 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000003-wal.1733343430826.temp to hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 at 1733343431060 (+1 ms)Processed 10 edits across 1 Regions in 34 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733343431060 2024-12-04T20:17:11,062 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343430826 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343430826 2024-12-04T20:17:11,063 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 2024-12-04T20:17:11,110 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:11,124 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-04T20:17:11,376 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:11,378 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:11,388 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343431378, exclude list is [], retry=0 2024-12-04T20:17:11,390 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:11,391 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:11,391 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:11,412 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343431378 2024-12-04T20:17:11,413 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:17:11,413 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dce577b5f5a6f191c56d001f4e870372, NAME => 'testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:11,413 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:11,413 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,413 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,416 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,417 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce577b5f5a6f191c56d001f4e870372 columnFamilyName a 2024-12-04T20:17:11,417 DEBUG [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,418 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(327): Store=dce577b5f5a6f191c56d001f4e870372/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,419 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,420 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,421 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 2024-12-04T20:17:11,446 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:11,447 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 2024-12-04T20:17:11,447 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dce577b5f5a6f191c56d001f4e870372 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-04T20:17:11,468 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/.tmp/a/36aad8c392be4066b537d3bc250f9afe is 79, key is testDatalossWhenInputError/a:x0/1733343430876/Put/seqid=0 2024-12-04T20:17:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741914_1092 (size=5808) 2024-12-04T20:17:11,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741914_1092 (size=5808) 2024-12-04T20:17:11,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741914_1092 (size=5808) 2024-12-04T20:17:11,476 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/.tmp/a/36aad8c392be4066b537d3bc250f9afe 2024-12-04T20:17:11,489 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/.tmp/a/36aad8c392be4066b537d3bc250f9afe as hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/a/36aad8c392be4066b537d3bc250f9afe 2024-12-04T20:17:11,499 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/a/36aad8c392be4066b537d3bc250f9afe, entries=10, sequenceid=12, filesize=5.7 K 2024-12-04T20:17:11,499 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dce577b5f5a6f191c56d001f4e870372 in 52ms, sequenceid=12, compaction requested=false; wal=null 2024-12-04T20:17:11,500 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/0000000000000000012 2024-12-04T20:17:11,502 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,502 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,505 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,507 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-04T20:17:11,508 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dce577b5f5a6f191c56d001f4e870372; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65158943, jitterRate=-0.029056087136268616}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:17:11,508 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dce577b5f5a6f191c56d001f4e870372: Writing region info on filesystem at 1733343431413Initializing all the Stores at 1733343431415 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431416 (+1 ms)Obtaining lock to block concurrent updates at 1733343431447 (+31 ms)Preparing flush snapshotting stores in dce577b5f5a6f191c56d001f4e870372 at 1733343431447Finished memstore snapshotting testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733343431447Flushing stores of testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372. at 1733343431447Flushing dce577b5f5a6f191c56d001f4e870372/a: creating writer at 1733343431447Flushing dce577b5f5a6f191c56d001f4e870372/a: appending metadata at 1733343431467 (+20 ms)Flushing dce577b5f5a6f191c56d001f4e870372/a: closing flushed file at 1733343431467Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f269d39: reopening flushed file at 1733343431487 (+20 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for dce577b5f5a6f191c56d001f4e870372 in 52ms, sequenceid=12, compaction requested=false; wal=null at 1733343431499 (+12 ms)Cleaning up temporary data from old regions at 1733343431502 (+3 ms)Region opened successfully at 1733343431508 (+6 ms) 2024-12-04T20:17:11,512 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dce577b5f5a6f191c56d001f4e870372, NAME => 'testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:11,512 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733343430749.dce577b5f5a6f191c56d001f4e870372.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:11,512 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,512 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,514 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,515 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dce577b5f5a6f191c56d001f4e870372 columnFamilyName a 2024-12-04T20:17:11,515 DEBUG [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,521 DEBUG [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/a/36aad8c392be4066b537d3bc250f9afe 2024-12-04T20:17:11,521 INFO [StoreOpener-dce577b5f5a6f191c56d001f4e870372-1 {}] regionserver.HStore(327): Store=dce577b5f5a6f191c56d001f4e870372/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,522 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,523 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,524 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,525 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,525 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,527 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dce577b5f5a6f191c56d001f4e870372 2024-12-04T20:17:11,530 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testDatalossWhenInputError/dce577b5f5a6f191c56d001f4e870372/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-04T20:17:11,531 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dce577b5f5a6f191c56d001f4e870372; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59170168, jitterRate=-0.11829578876495361}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-04T20:17:11,532 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dce577b5f5a6f191c56d001f4e870372: Writing region info on filesystem at 1733343431512Initializing all the Stores at 1733343431513 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431514 (+1 ms)Cleaning up temporary data from old regions at 1733343431525 (+11 ms)Region opened successfully at 1733343431532 (+7 ms) 2024-12-04T20:17:11,548 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=424 (was 415) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36148 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:42970 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36046 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:42950 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:47980 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:48066 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 148) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4922 (was 4946) 2024-12-04T20:17:11,548 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-04T20:17:11,559 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=424, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=4921 2024-12-04T20:17:11,559 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-04T20:17:11,571 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:11,572 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:11,573 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:17:11,575 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-54110736, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-54110736, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:11,586 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-54110736/hregion-54110736.1733343431576, exclude list is [], retry=0 2024-12-04T20:17:11,589 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:11,589 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:11,589 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:11,616 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-54110736/hregion-54110736.1733343431576 2024-12-04T20:17:11,616 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:17:11,616 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4d3cef6fa2033b507272e9276603e3e5, NAME => 'testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:17:11,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741916_1094 (size=63) 2024-12-04T20:17:11,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741916_1094 (size=63) 2024-12-04T20:17:11,628 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741916_1094 (size=63) 2024-12-04T20:17:11,629 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:11,631 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,632 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName a 2024-12-04T20:17:11,632 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,633 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,633 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,634 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName b 2024-12-04T20:17:11,634 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,635 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,635 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,636 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName c 2024-12-04T20:17:11,636 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,637 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,637 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,638 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,639 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,639 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,640 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:11,641 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,643 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:17:11,643 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d3cef6fa2033b507272e9276603e3e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65874441, jitterRate=-0.018394336104393005}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d3cef6fa2033b507272e9276603e3e5: Writing region info on filesystem at 1733343431629Initializing all the Stores at 1733343431630 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431630Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431631 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431631Cleaning up temporary data from old regions at 1733343431639 (+8 ms)Region opened successfully at 1733343431644 (+5 ms) 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4d3cef6fa2033b507272e9276603e3e5, disabling compactions & flushes 2024-12-04T20:17:11,644 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. after waiting 0 ms 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:11,644 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:11,644 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4d3cef6fa2033b507272e9276603e3e5: Waiting for close lock at 1733343431644Disabling compacts and flushes for region at 1733343431644Disabling writes for close at 1733343431644Writing region close event to WAL at 1733343431644Closed at 1733343431644 2024-12-04T20:17:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741915_1093 (size=175) 2024-12-04T20:17:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741915_1093 (size=175) 2024-12-04T20:17:11,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741915_1093 (size=175) 2024-12-04T20:17:11,652 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:11,652 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-54110736:(num 1733343431576) 2024-12-04T20:17:11,652 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:11,655 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:11,666 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, exclude list is [], retry=0 2024-12-04T20:17:11,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:11,669 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:11,670 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:11,692 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 2024-12-04T20:17:11,692 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37929:37929)] 2024-12-04T20:17:11,693 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d3cef6fa2033b507272e9276603e3e5, NAME => 'testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:11,693 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:11,693 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,693 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,694 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,695 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName a 2024-12-04T20:17:11,695 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,696 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,696 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,696 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName b 2024-12-04T20:17:11,696 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,697 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,697 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,697 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName c 2024-12-04T20:17:11,698 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:11,698 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:11,698 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,699 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,700 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,701 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,701 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,701 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:11,702 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:11,703 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d3cef6fa2033b507272e9276603e3e5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69146358, jitterRate=0.030361026525497437}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:11,704 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d3cef6fa2033b507272e9276603e3e5: Writing region info on filesystem at 1733343431693Initializing all the Stores at 1733343431694 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431694Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431694Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343431694Cleaning up temporary data from old regions at 1733343431701 (+7 ms)Region opened successfully at 1733343431704 (+3 ms) 2024-12-04T20:17:11,707 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733343431707/Put/seqid=0 2024-12-04T20:17:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741918_1096 (size=4875) 2024-12-04T20:17:11,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741918_1096 (size=4875) 2024-12-04T20:17:11,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741918_1096 (size=4875) 2024-12-04T20:17:11,716 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733343431715/Put/seqid=0 2024-12-04T20:17:11,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741919_1097 (size=4875) 2024-12-04T20:17:11,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741919_1097 (size=4875) 2024-12-04T20:17:11,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741919_1097 (size=4875) 2024-12-04T20:17:11,730 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733343431730/Put/seqid=0 2024-12-04T20:17:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741920_1098 (size=4875) 2024-12-04T20:17:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741920_1098 (size=4875) 2024-12-04T20:17:11,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741920_1098 (size=4875) 2024-12-04T20:17:11,739 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,742 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-04T20:17:11,742 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T20:17:11,742 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,746 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-04T20:17:11,746 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T20:17:11,746 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,750 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-04T20:17:11,750 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-04T20:17:11,751 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4d3cef6fa2033b507272e9276603e3e5 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-04T20:17:11,765 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp/a/51b188d585f84be4a26fc7edfd86f254 is 55, key is testCompactedBulkLoadedFiles/a:a/1733343431704/Put/seqid=0 2024-12-04T20:17:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741921_1099 (size=5107) 2024-12-04T20:17:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741921_1099 (size=5107) 2024-12-04T20:17:11,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741921_1099 (size=5107) 2024-12-04T20:17:11,773 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp/a/51b188d585f84be4a26fc7edfd86f254 2024-12-04T20:17:11,778 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp/a/51b188d585f84be4a26fc7edfd86f254 as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254 2024-12-04T20:17:11,783 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254, entries=1, sequenceid=4, filesize=5.0 K 2024-12-04T20:17:11,785 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 4d3cef6fa2033b507272e9276603e3e5 in 35ms, sequenceid=4, compaction requested=false 2024-12-04T20:17:11,785 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 4d3cef6fa2033b507272e9276603e3e5: 2024-12-04T20:17:11,786 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ 2024-12-04T20:17:11,787 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ 2024-12-04T20:17:11,788 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ 2024-12-04T20:17:11,789 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile0 into 4d3cef6fa2033b507272e9276603e3e5/a as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ - updating store file list. 2024-12-04T20:17:11,793 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1306aefce15c48609a90c96b688a3db5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:11,793 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ into 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,793 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile0 into 4d3cef6fa2033b507272e9276603e3e5/a (new location: hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_) 2024-12-04T20:17:11,794 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile1 into 4d3cef6fa2033b507272e9276603e3e5/a as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ - updating store file list. 2024-12-04T20:17:11,799 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 772132cf89d24ba489e06982abda6488_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:11,799 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ into 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,799 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile1 into 4d3cef6fa2033b507272e9276603e3e5/a (new location: hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_) 2024-12-04T20:17:11,800 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile2 into 4d3cef6fa2033b507272e9276603e3e5/a as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ - updating store file list. 2024-12-04T20:17:11,805 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e1d116c40caa488f806241246cbaee47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:11,805 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ into 4d3cef6fa2033b507272e9276603e3e5/a 2024-12-04T20:17:11,805 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:44433/hbase/testCompactedBulkLoadedFiles/hfile2 into 4d3cef6fa2033b507272e9276603e3e5/a (new location: hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_) 2024-12-04T20:17:11,812 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-04T20:17:11,812 DEBUG [Time-limited test {}] regionserver.HStore(1541): 4d3cef6fa2033b507272e9276603e3e5/a is initiating major compaction (all files) 2024-12-04T20:17:11,812 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 4d3cef6fa2033b507272e9276603e3e5/a in testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:11,813 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_] into tmpdir=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp, totalSize=19.3 K 2024-12-04T20:17:11,813 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 51b188d585f84be4a26fc7edfd86f254, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733343431704 2024-12-04T20:17:11,814 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1306aefce15c48609a90c96b688a3db5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T20:17:11,814 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 772132cf89d24ba489e06982abda6488_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T20:17:11,814 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e1d116c40caa488f806241246cbaee47_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-04T20:17:11,826 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp/a/3ed473b4e87845089e20aaefafacc3c9 is 55, key is testCompactedBulkLoadedFiles/a:a/1733343431704/Put/seqid=0 2024-12-04T20:17:11,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741922_1100 (size=6154) 2024-12-04T20:17:11,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741922_1100 (size=6154) 2024-12-04T20:17:11,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741922_1100 (size=6154) 2024-12-04T20:17:11,840 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/.tmp/a/3ed473b4e87845089e20aaefafacc3c9 as hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/3ed473b4e87845089e20aaefafacc3c9 2024-12-04T20:17:11,847 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 4d3cef6fa2033b507272e9276603e3e5/a of 4d3cef6fa2033b507272e9276603e3e5 into 3ed473b4e87845089e20aaefafacc3c9(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-04T20:17:11,847 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 4d3cef6fa2033b507272e9276603e3e5: 2024-12-04T20:17:11,847 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T20:17:11,847 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-04T20:17:11,879 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, size=0 (0bytes) 2024-12-04T20:17:11,879 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 might be still open, length is 0 2024-12-04T20:17:11,879 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 2024-12-04T20:17:11,880 WARN [IPC Server handler 4 on default port 44433 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-04T20:17:11,880 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 after 1ms 2024-12-04T20:17:12,301 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-04T20:17:12,301 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:12,302 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-04T20:17:12,302 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-04T20:17:13,963 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:48118 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:42009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48118 dst: /127.0.0.1:42009 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:42009 remote=/127.0.0.1:48118]. Total timeout mills is 60000, 57884 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:13,964 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:43004 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:37737:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43004 dst: /127.0.0.1:37737 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:13,964 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36182 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:38071:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36182 dst: /127.0.0.1:38071 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:13,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741917_1101 (size=1794) 2024-12-04T20:17:13,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741917_1101 (size=1794) 2024-12-04T20:17:13,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741917_1101 (size=1794) 2024-12-04T20:17:15,881 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 after 4002ms 2024-12-04T20:17:15,912 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:15,913 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 took 4034ms 2024-12-04T20:17:15,914 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655; continuing. 2024-12-04T20:17:15,914 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 so closing down 2024-12-04T20:17:15,915 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:15,916 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733343431655.temp 2024-12-04T20:17:15,932 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-04T20:17:15,938 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp 2024-12-04T20:17:15,939 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:15,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741923_1102 (size=800) 2024-12-04T20:17:15,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741923_1102 (size=800) 2024-12-04T20:17:15,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741923_1102 (size=800) 2024-12-04T20:17:15,947 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:15,948 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp to hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 2024-12-04T20:17:15,949 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T20:17:15,949 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, journal: Splitting hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, size=0 (0bytes) at 1733343431879Finishing writing output for hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 so closing down at 1733343435914 (+4035 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp at 1733343435939 (+25 ms)3 split writer threads finished at 1733343435939Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733343435947 (+8 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000003-wal.1733343431655.temp to hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 at 1733343435948 (+1 ms)Processed 5 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655, size=0, length=0, corrupted=false, cancelled=false at 1733343435949 (+1 ms) 2024-12-04T20:17:15,950 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343431655 2024-12-04T20:17:15,951 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 2024-12-04T20:17:15,951 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:15,953 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:15,964 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343435953, exclude list is [], retry=0 2024-12-04T20:17:15,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:15,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:15,966 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:15,987 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343435953 2024-12-04T20:17:15,987 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:17:15,987 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4d3cef6fa2033b507272e9276603e3e5, NAME => 'testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:15,987 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:15,988 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:15,989 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:15,990 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName a 2024-12-04T20:17:15,990 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:15,996 DEBUG [StoreFileOpener-4d3cef6fa2033b507272e9276603e3e5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1306aefce15c48609a90c96b688a3db5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:15,996 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ 2024-12-04T20:17:16,000 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/3ed473b4e87845089e20aaefafacc3c9 2024-12-04T20:17:16,004 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254 2024-12-04T20:17:16,007 DEBUG [StoreFileOpener-4d3cef6fa2033b507272e9276603e3e5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 772132cf89d24ba489e06982abda6488_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:16,007 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ 2024-12-04T20:17:16,010 DEBUG [StoreFileOpener-4d3cef6fa2033b507272e9276603e3e5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e1d116c40caa488f806241246cbaee47_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-04T20:17:16,011 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ 2024-12-04T20:17:16,011 WARN [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@8259ef6 2024-12-04T20:17:16,011 WARN [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@8259ef6 2024-12-04T20:17:16,011 WARN [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@8259ef6 2024-12-04T20:17:16,011 WARN [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@8259ef6 2024-12-04T20:17:16,011 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_] to archive 2024-12-04T20:17:16,012 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-04T20:17:16,014 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ to hdfs://localhost:44433/hbase/archive/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/1306aefce15c48609a90c96b688a3db5_SeqId_4_ 2024-12-04T20:17:16,015 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254 to hdfs://localhost:44433/hbase/archive/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/51b188d585f84be4a26fc7edfd86f254 2024-12-04T20:17:16,016 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ to hdfs://localhost:44433/hbase/archive/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/772132cf89d24ba489e06982abda6488_SeqId_4_ 2024-12-04T20:17:16,017 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ to hdfs://localhost:44433/hbase/archive/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/a/e1d116c40caa488f806241246cbaee47_SeqId_4_ 2024-12-04T20:17:16,017 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,017 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,018 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName b 2024-12-04T20:17:16,018 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,018 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,018 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,019 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4d3cef6fa2033b507272e9276603e3e5 columnFamilyName c 2024-12-04T20:17:16,019 DEBUG [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,020 INFO [StoreOpener-4d3cef6fa2033b507272e9276603e3e5-1 {}] regionserver.HStore(327): Store=4d3cef6fa2033b507272e9276603e3e5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,020 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,021 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,022 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,023 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 2024-12-04T20:17:16,055 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:16,056 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 4d3cef6fa2033b507272e9276603e3e5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "4d3cef6fa2033b507272e9276603e3e5" family_name: "a" compaction_input: "51b188d585f84be4a26fc7edfd86f254" compaction_input: "1306aefce15c48609a90c96b688a3db5_SeqId_4_" compaction_input: "772132cf89d24ba489e06982abda6488_SeqId_4_" compaction_input: "e1d116c40caa488f806241246cbaee47_SeqId_4_" compaction_output: "3ed473b4e87845089e20aaefafacc3c9" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-04T20:17:16,056 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-04T20:17:16,057 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 2024-12-04T20:17:16,057 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/0000000000000000008 2024-12-04T20:17:16,059 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,059 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,059 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:16,060 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4d3cef6fa2033b507272e9276603e3e5 2024-12-04T20:17:16,063 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testCompactedBulkLoadedFiles/4d3cef6fa2033b507272e9276603e3e5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-04T20:17:16,064 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4d3cef6fa2033b507272e9276603e3e5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62573164, jitterRate=-0.06758719682693481}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:16,064 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4d3cef6fa2033b507272e9276603e3e5: Writing region info on filesystem at 1733343435988Initializing all the Stores at 1733343435989 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343435989Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343435989Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343435989Cleaning up temporary data from old regions at 1733343436059 (+70 ms)Region opened successfully at 1733343436064 (+5 ms) 2024-12-04T20:17:16,066 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4d3cef6fa2033b507272e9276603e3e5, disabling compactions & flushes 2024-12-04T20:17:16,066 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:16,066 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:16,066 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. after waiting 0 ms 2024-12-04T20:17:16,066 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:16,067 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733343431571.4d3cef6fa2033b507272e9276603e3e5. 2024-12-04T20:17:16,067 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4d3cef6fa2033b507272e9276603e3e5: Waiting for close lock at 1733343436066Disabling compacts and flushes for region at 1733343436066Disabling writes for close at 1733343436066Writing region close event to WAL at 1733343436067 (+1 ms)Closed at 1733343436067 2024-12-04T20:17:16,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741924_1103 (size=175) 2024-12-04T20:17:16,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741924_1103 (size=175) 2024-12-04T20:17:16,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741924_1103 (size=175) 2024-12-04T20:17:16,071 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:16,071 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733343435953) 2024-12-04T20:17:16,083 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=439 (was 424) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44433 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_330168737_22 at /127.0.0.1:48214 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:44433 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_330168737_22 at /127.0.0.1:36212 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_330168737_22 at /127.0.0.1:43066 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=208 (was 208), ProcessCount=11 (was 11), AvailableMemoryMB=4925 (was 4921) - AvailableMemoryMB LEAK? - 2024-12-04T20:17:16,083 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-04T20:17:16,096 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=208, ProcessCount=11, AvailableMemoryMB=4924 2024-12-04T20:17:16,096 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-12-04T20:17:16,112 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:16,114 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-04T20:17:16,114 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-04T20:17:16,117 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-07979484, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/hregion-07979484, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:16,129 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-07979484/hregion-07979484.1733343436117, exclude list is [], retry=0 2024-12-04T20:17:16,131 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:16,132 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:16,132 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:16,156 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-07979484/hregion-07979484.1733343436117 2024-12-04T20:17:16,156 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:16,156 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3639d89450ab6184ab50ee8682024109, NAME => 'testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:44433/hbase 2024-12-04T20:17:16,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741926_1105 (size=67) 2024-12-04T20:17:16,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741926_1105 (size=67) 2024-12-04T20:17:16,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741926_1105 (size=67) 2024-12-04T20:17:16,165 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:16,167 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,168 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName a 2024-12-04T20:17:16,168 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,168 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,168 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,169 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName b 2024-12-04T20:17:16,170 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,170 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,170 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,171 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName c 2024-12-04T20:17:16,171 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,172 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,172 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,173 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,174 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,174 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,175 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:16,176 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,178 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-04T20:17:16,179 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3639d89450ab6184ab50ee8682024109; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69562084, jitterRate=0.03655582666397095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:16,179 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3639d89450ab6184ab50ee8682024109: Writing region info on filesystem at 1733343436165Initializing all the Stores at 1733343436166 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436166Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436166Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436166Cleaning up temporary data from old regions at 1733343436174 (+8 ms)Region opened successfully at 1733343436179 (+5 ms) 2024-12-04T20:17:16,179 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3639d89450ab6184ab50ee8682024109, disabling compactions & flushes 2024-12-04T20:17:16,179 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,179 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,179 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. after waiting 0 ms 2024-12-04T20:17:16,179 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,180 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,180 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3639d89450ab6184ab50ee8682024109: Waiting for close lock at 1733343436179Disabling compacts and flushes for region at 1733343436179Disabling writes for close at 1733343436179Writing region close event to WAL at 1733343436180 (+1 ms)Closed at 1733343436180 2024-12-04T20:17:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741925_1104 (size=175) 2024-12-04T20:17:16,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741925_1104 (size=175) 2024-12-04T20:17:16,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741925_1104 (size=175) 2024-12-04T20:17:16,184 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:16,184 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-07979484:(num 1733343436117) 2024-12-04T20:17:16,184 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:16,186 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:16,198 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, exclude list is [], retry=0 2024-12-04T20:17:16,201 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:16,201 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:16,201 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:16,225 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 2024-12-04T20:17:16,225 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:16,225 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3639d89450ab6184ab50ee8682024109, NAME => 'testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:16,226 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:16,226 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,226 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,227 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,228 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName a 2024-12-04T20:17:16,228 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,229 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,229 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,230 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName b 2024-12-04T20:17:16,230 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,231 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,231 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,232 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName c 2024-12-04T20:17:16,232 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,232 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,232 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,233 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,235 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,236 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,236 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,236 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:16,238 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,239 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3639d89450ab6184ab50ee8682024109; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72856211, jitterRate=0.08564214408397675}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:16,239 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3639d89450ab6184ab50ee8682024109: Writing region info on filesystem at 1733343436226Initializing all the Stores at 1733343436227 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436227Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436227Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436227Cleaning up temporary data from old regions at 1733343436236 (+9 ms)Region opened successfully at 1733343436239 (+3 ms) 2024-12-04T20:17:16,246 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3639d89450ab6184ab50ee8682024109 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-04T20:17:16,259 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/7ee15d0e5b944424b332719b8e71f415 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733343436239/Put/seqid=0 2024-12-04T20:17:16,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741928_1107 (size=5958) 2024-12-04T20:17:16,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741928_1107 (size=5958) 2024-12-04T20:17:16,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741928_1107 (size=5958) 2024-12-04T20:17:16,265 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/7ee15d0e5b944424b332719b8e71f415 2024-12-04T20:17:16,270 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/7ee15d0e5b944424b332719b8e71f415 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/7ee15d0e5b944424b332719b8e71f415 2024-12-04T20:17:16,276 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/7ee15d0e5b944424b332719b8e71f415, entries=10, sequenceid=13, filesize=5.8 K 2024-12-04T20:17:16,277 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 3639d89450ab6184ab50ee8682024109 in 31ms, sequenceid=13, compaction requested=false 2024-12-04T20:17:16,277 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3639d89450ab6184ab50ee8682024109: 2024-12-04T20:17:16,294 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3639d89450ab6184ab50ee8682024109, disabling compactions & flushes 2024-12-04T20:17:16,294 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,294 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,294 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. after waiting 0 ms 2024-12-04T20:17:16,294 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,295 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,295 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:16,295 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3639d89450ab6184ab50ee8682024109: Waiting for close lock at 1733343436294Disabling compacts and flushes for region at 1733343436294Disabling writes for close at 1733343436294Writing region close event to WAL at 1733343436295 (+1 ms)Closed at 1733343436295 2024-12-04T20:17:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741927_1106 (size=6243) 2024-12-04T20:17:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741927_1106 (size=6243) 2024-12-04T20:17:16,298 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741927_1106 (size=6243) 2024-12-04T20:17:16,311 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, size=6.1 K (6243bytes) 2024-12-04T20:17:16,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 2024-12-04T20:17:16,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 after 0ms 2024-12-04T20:17:16,335 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:16,335 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 took 24ms 2024-12-04T20:17:16,337 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 so closing down 2024-12-04T20:17:16,338 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:16,339 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733343436187.temp 2024-12-04T20:17:16,361 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp 2024-12-04T20:17:16,361 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:16,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741929_1108 (size=5575) 2024-12-04T20:17:16,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741929_1108 (size=5575) 2024-12-04T20:17:16,368 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741929_1108 (size=5575) 2024-12-04T20:17:16,368 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:16,370 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 2024-12-04T20:17:16,370 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-04T20:17:16,370 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, journal: Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, size=6.1 K (6243bytes) at 1733343436312Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 so closing down at 1733343436338 (+26 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp at 1733343436361 (+23 ms)3 split writer threads finished at 1733343436361Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733343436368 (+7 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000003-wal.1733343436187.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 at 1733343436370 (+2 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733343436370 2024-12-04T20:17:16,372 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436187 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343436187 2024-12-04T20:17:16,373 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 2024-12-04T20:17:16,373 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:16,374 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:16,393 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, exclude list is [], retry=0 2024-12-04T20:17:16,395 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:16,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:16,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:16,420 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 2024-12-04T20:17:16,420 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367),(127.0.0.1/127.0.0.1:37901:37901)] 2024-12-04T20:17:16,420 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3639d89450ab6184ab50ee8682024109, NAME => 'testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.', STARTKEY => '', ENDKEY => ''} 2024-12-04T20:17:16,420 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:16,420 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,420 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,422 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,422 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName a 2024-12-04T20:17:16,423 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,428 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/7ee15d0e5b944424b332719b8e71f415 2024-12-04T20:17:16,428 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,428 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,429 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName b 2024-12-04T20:17:16,429 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,430 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,430 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,431 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName c 2024-12-04T20:17:16,431 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:16,431 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:16,431 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,432 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,433 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,433 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 2024-12-04T20:17:16,455 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:16,456 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 2024-12-04T20:17:16,456 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3639d89450ab6184ab50ee8682024109 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-04T20:17:16,468 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/bd9af725b45c4dff992e59a0fb2b081d is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733343436277/Put/seqid=0 2024-12-04T20:17:16,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741931_1110 (size=5958) 2024-12-04T20:17:16,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741931_1110 (size=5958) 2024-12-04T20:17:16,474 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741931_1110 (size=5958) 2024-12-04T20:17:16,474 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/bd9af725b45c4dff992e59a0fb2b081d 2024-12-04T20:17:16,491 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/c391ec93e97c4199ba1214f408f51d7c is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733343436285/Put/seqid=0 2024-12-04T20:17:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741932_1111 (size=5958) 2024-12-04T20:17:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741932_1111 (size=5958) 2024-12-04T20:17:16,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741932_1111 (size=5958) 2024-12-04T20:17:16,498 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/c391ec93e97c4199ba1214f408f51d7c 2024-12-04T20:17:16,503 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/bd9af725b45c4dff992e59a0fb2b081d as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/b/bd9af725b45c4dff992e59a0fb2b081d 2024-12-04T20:17:16,508 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/b/bd9af725b45c4dff992e59a0fb2b081d, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T20:17:16,509 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/c391ec93e97c4199ba1214f408f51d7c as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/c/c391ec93e97c4199ba1214f408f51d7c 2024-12-04T20:17:16,513 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/c/c391ec93e97c4199ba1214f408f51d7c, entries=10, sequenceid=35, filesize=5.8 K 2024-12-04T20:17:16,514 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3639d89450ab6184ab50ee8682024109 in 58ms, sequenceid=35, compaction requested=false; wal=null 2024-12-04T20:17:16,514 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000035 2024-12-04T20:17:16,515 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,516 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,516 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:16,518 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:16,520 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-04T20:17:16,521 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3639d89450ab6184ab50ee8682024109; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59234494, jitterRate=-0.11733725666999817}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:16,521 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3639d89450ab6184ab50ee8682024109: Writing region info on filesystem at 1733343436420Initializing all the Stores at 1733343436421 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436421Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436422 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343436422Obtaining lock to block concurrent updates at 1733343436456 (+34 ms)Preparing flush snapshotting stores in 3639d89450ab6184ab50ee8682024109 at 1733343436456Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733343436456Flushing stores of testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. at 1733343436456Flushing 3639d89450ab6184ab50ee8682024109/b: creating writer at 1733343436456Flushing 3639d89450ab6184ab50ee8682024109/b: appending metadata at 1733343436467 (+11 ms)Flushing 3639d89450ab6184ab50ee8682024109/b: closing flushed file at 1733343436467Flushing 3639d89450ab6184ab50ee8682024109/c: creating writer at 1733343436478 (+11 ms)Flushing 3639d89450ab6184ab50ee8682024109/c: appending metadata at 1733343436490 (+12 ms)Flushing 3639d89450ab6184ab50ee8682024109/c: closing flushed file at 1733343436490Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7e2bb660: reopening flushed file at 1733343436502 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@41a2025b: reopening flushed file at 1733343436508 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 3639d89450ab6184ab50ee8682024109 in 58ms, sequenceid=35, compaction requested=false; wal=null at 1733343436514 (+6 ms)Cleaning up temporary data from old regions at 1733343436516 (+2 ms)Region opened successfully at 1733343436521 (+5 ms) 2024-12-04T20:17:16,578 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, size=0 (0bytes) 2024-12-04T20:17:16,578 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 might be still open, length is 0 2024-12-04T20:17:16,578 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 2024-12-04T20:17:16,579 WARN [IPC Server handler 4 on default port 44433 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-04T20:17:16,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 after 1ms 2024-12-04T20:17:16,959 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:36264 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:38071:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36264 dst: /127.0.0.1:38071 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:38071 remote=/127.0.0.1:36264]. Total timeout mills is 60000, 59587 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:16,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:43094 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:37737:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:43094 dst: /127.0.0.1:37737 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:16,960 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1811373311_22 at /127.0.0.1:48254 [Receiving block BP-700201894-172.17.0.2-1733343399373:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:42009:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:48254 dst: /127.0.0.1:42009 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741930_1112 (size=5567) 2024-12-04T20:17:16,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741930_1112 (size=5567) 2024-12-04T20:17:16,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741930_1112 (size=5567) 2024-12-04T20:17:20,579 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 after 4001ms 2024-12-04T20:17:20,607 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:20,607 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 took 4029ms 2024-12-04T20:17:20,610 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375; continuing. 2024-12-04T20:17:20,610 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 so closing down 2024-12-04T20:17:20,610 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-04T20:17:20,611 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733343436375.temp 2024-12-04T20:17:20,632 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp 2024-12-04T20:17:20,632 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-04T20:17:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741933_1113 (size=5575) 2024-12-04T20:17:20,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741933_1113 (size=5575) 2024-12-04T20:17:20,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741933_1113 (size=5575) 2024-12-04T20:17:20,638 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-04T20:17:20,639 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 2024-12-04T20:17:20,639 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, size=0, length=0, corrupted=false, cancelled=false 2024-12-04T20:17:20,639 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, journal: Splitting hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, size=0 (0bytes) at 1733343436578Finishing writing output for hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 so closing down at 1733343440610 (+4032 ms)Creating recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp at 1733343440632 (+22 ms)3 split writer threads finished at 1733343440632Closed recovered edits writer path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733343440638 (+6 ms)Rename recovered edits hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000037-wal.1733343436375.temp to hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 at 1733343440639 (+1 ms)Processed 30 edits across 1 Regions in 31 ms; skipped=0; WAL=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375, size=0, length=0, corrupted=false, cancelled=false at 1733343440639 2024-12-04T20:17:20,641 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 to hdfs://localhost:44433/hbase/oldWALs/wal.1733343436375 2024-12-04T20:17:20,642 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 2024-12-04T20:17:20,642 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-04T20:17:20,643 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:44433/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111, archiveDir=hdfs://localhost:44433/hbase/oldWALs, maxLogs=32 2024-12-04T20:17:20,656 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343440644, exclude list is [], retry=0 2024-12-04T20:17:20,658 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42009,DS-f76ad4c4-1f09-4138-a8b3-3452281f0ab2,DISK] 2024-12-04T20:17:20,658 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38071,DS-b5812768-870d-43ac-b12c-85ccd56a7c36,DISK] 2024-12-04T20:17:20,658 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37737,DS-1b73fe1e-318e-4ddf-9174-f73ff5d3c351,DISK] 2024-12-04T20:17:20,681 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343440644 2024-12-04T20:17:20,681 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37901:37901),(127.0.0.1/127.0.0.1:37929:37929),(127.0.0.1/127.0.0.1:44367:44367)] 2024-12-04T20:17:20,681 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-04T20:17:20,683 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,684 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName a 2024-12-04T20:17:20,684 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:20,688 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/7ee15d0e5b944424b332719b8e71f415 2024-12-04T20:17:20,688 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:20,688 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,689 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName b 2024-12-04T20:17:20,689 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:20,694 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/b/bd9af725b45c4dff992e59a0fb2b081d 2024-12-04T20:17:20,694 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:20,695 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,696 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3639d89450ab6184ab50ee8682024109 columnFamilyName c 2024-12-04T20:17:20,696 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-04T20:17:20,701 DEBUG [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/c/c391ec93e97c4199ba1214f408f51d7c 2024-12-04T20:17:20,701 INFO [StoreOpener-3639d89450ab6184ab50ee8682024109-1 {}] regionserver.HStore(327): Store=3639d89450ab6184ab50ee8682024109/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-04T20:17:20,701 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,702 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,704 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,704 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 2024-12-04T20:17:20,739 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-04T20:17:20,743 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 2024-12-04T20:17:20,743 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3639d89450ab6184ab50ee8682024109 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-04T20:17:20,756 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/92d8c0db6f624c3d82d1a20164cff761 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733343436529/Put/seqid=0 2024-12-04T20:17:20,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741935_1115 (size=5958) 2024-12-04T20:17:20,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741935_1115 (size=5958) 2024-12-04T20:17:20,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741935_1115 (size=5958) 2024-12-04T20:17:20,764 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/92d8c0db6f624c3d82d1a20164cff761 2024-12-04T20:17:20,782 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/645d5032ee7247f893228cf76cb6b3fa is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733343436536/Put/seqid=0 2024-12-04T20:17:20,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741936_1116 (size=5958) 2024-12-04T20:17:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741936_1116 (size=5958) 2024-12-04T20:17:20,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741936_1116 (size=5958) 2024-12-04T20:17:20,788 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/645d5032ee7247f893228cf76cb6b3fa 2024-12-04T20:17:20,806 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/7c8587fad9e345d3a8849d2fc8266aa5 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733343436541/Put/seqid=0 2024-12-04T20:17:20,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741937_1117 (size=5958) 2024-12-04T20:17:20,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741937_1117 (size=5958) 2024-12-04T20:17:20,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741937_1117 (size=5958) 2024-12-04T20:17:20,812 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/7c8587fad9e345d3a8849d2fc8266aa5 2024-12-04T20:17:20,818 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/a/92d8c0db6f624c3d82d1a20164cff761 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/92d8c0db6f624c3d82d1a20164cff761 2024-12-04T20:17:20,823 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/a/92d8c0db6f624c3d82d1a20164cff761, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T20:17:20,824 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/b/645d5032ee7247f893228cf76cb6b3fa as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/b/645d5032ee7247f893228cf76cb6b3fa 2024-12-04T20:17:20,829 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/b/645d5032ee7247f893228cf76cb6b3fa, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T20:17:20,833 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/.tmp/c/7c8587fad9e345d3a8849d2fc8266aa5 as hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/c/7c8587fad9e345d3a8849d2fc8266aa5 2024-12-04T20:17:20,839 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/c/7c8587fad9e345d3a8849d2fc8266aa5, entries=10, sequenceid=66, filesize=5.8 K 2024-12-04T20:17:20,839 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3639d89450ab6184ab50ee8682024109 in 96ms, sequenceid=66, compaction requested=false; wal=null 2024-12-04T20:17:20,840 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/0000000000000000066 2024-12-04T20:17:20,841 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,841 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,842 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-04T20:17:20,843 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3639d89450ab6184ab50ee8682024109 2024-12-04T20:17:20,846 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/hbase/data/default/testReplayEditsWrittenViaHRegion/3639d89450ab6184ab50ee8682024109/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-04T20:17:20,847 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3639d89450ab6184ab50ee8682024109; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72638051, jitterRate=0.08239130675792694}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-04T20:17:20,847 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3639d89450ab6184ab50ee8682024109: Writing region info on filesystem at 1733343440682Initializing all the Stores at 1733343440682Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343440682Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343440683 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733343440683Obtaining lock to block concurrent updates at 1733343440743 (+60 ms)Preparing flush snapshotting stores in 3639d89450ab6184ab50ee8682024109 at 1733343440743Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733343440743Flushing stores of testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. at 1733343440743Flushing 3639d89450ab6184ab50ee8682024109/a: creating writer at 1733343440743Flushing 3639d89450ab6184ab50ee8682024109/a: appending metadata at 1733343440755 (+12 ms)Flushing 3639d89450ab6184ab50ee8682024109/a: closing flushed file at 1733343440755Flushing 3639d89450ab6184ab50ee8682024109/b: creating writer at 1733343440768 (+13 ms)Flushing 3639d89450ab6184ab50ee8682024109/b: appending metadata at 1733343440781 (+13 ms)Flushing 3639d89450ab6184ab50ee8682024109/b: closing flushed file at 1733343440781Flushing 3639d89450ab6184ab50ee8682024109/c: creating writer at 1733343440793 (+12 ms)Flushing 3639d89450ab6184ab50ee8682024109/c: appending metadata at 1733343440806 (+13 ms)Flushing 3639d89450ab6184ab50ee8682024109/c: closing flushed file at 1733343440806Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2cd36cf7: reopening flushed file at 1733343440817 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@84f45c8: reopening flushed file at 1733343440823 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6b882f4f: reopening flushed file at 1733343440829 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 3639d89450ab6184ab50ee8682024109 in 96ms, sequenceid=66, compaction requested=false; wal=null at 1733343440839 (+10 ms)Cleaning up temporary data from old regions at 1733343440841 (+2 ms)Region opened successfully at 1733343440847 (+6 ms) 2024-12-04T20:17:20,860 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3639d89450ab6184ab50ee8682024109, disabling compactions & flushes 2024-12-04T20:17:20,860 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:20,860 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:20,861 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. after waiting 0 ms 2024-12-04T20:17:20,861 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:20,862 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733343436112.3639d89450ab6184ab50ee8682024109. 2024-12-04T20:17:20,862 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3639d89450ab6184ab50ee8682024109: Waiting for close lock at 1733343440860Disabling compacts and flushes for region at 1733343440860Disabling writes for close at 1733343440861 (+1 ms)Writing region close event to WAL at 1733343440862 (+1 ms)Closed at 1733343440862 2024-12-04T20:17:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741934_1114 (size=175) 2024-12-04T20:17:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741934_1114 (size=175) 2024-12-04T20:17:20,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741934_1114 (size=175) 2024-12-04T20:17:20,866 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-04T20:17:20,866 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733343440644) 2024-12-04T20:17:20,879 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=444 (was 439) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_96866928_22 at /127.0.0.1:53276 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_96866928_22 at /127.0.0.1:36088 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_96866928_22 at /127.0.0.1:47652 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:44433 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:44433 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=215 (was 208) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=4912 (was 4924) 2024-12-04T20:17:20,879 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-04T20:17:20,879 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-04T20:17:20,879 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:17:20,879 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:17:20,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:20,880 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:20,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-04T20:17:20,880 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-04T20:17:20,881 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=402235364, stopped=false 2024-12-04T20:17:20,881 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=c2ef38372881,37947,1733343401962 2024-12-04T20:17:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:17:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:17:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:17:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-04T20:17:20,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:17:20,883 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:17:20,883 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:17:20,883 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:17:20,883 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:17:20,883 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-04T20:17:20,884 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-04T20:17:20,884 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:17:20,884 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:20,884 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,45313,1733343402534' ***** 2024-12-04T20:17:20,884 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:17:20,884 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'c2ef38372881,40377,1733343402660' ***** 2024-12-04T20:17:20,884 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:17:20,885 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:17:20,885 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,40377,1733343402660 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(3091): Received CLOSE for 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;c2ef38372881:40377. 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(959): stopping server c2ef38372881,45313,1733343402534 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:17:20,885 DEBUG [RS:2;c2ef38372881:40377 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:17:20,885 DEBUG [RS:2;c2ef38372881:40377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;c2ef38372881:45313. 2024-12-04T20:17:20,885 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 398cfc7d795642642bde6a262384a0b3, disabling compactions & flushes 2024-12-04T20:17:20,885 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:20,885 DEBUG [RS:0;c2ef38372881:45313 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-04T20:17:20,885 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:20,885 DEBUG [RS:0;c2ef38372881:45313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:20,885 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. after waiting 0 ms 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:17:20,885 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:17:20,885 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:17:20,885 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1325): Online Regions={398cfc7d795642642bde6a262384a0b3=testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3.} 2024-12-04T20:17:20,885 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-04T20:17:20,885 DEBUG [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1351): Waiting on 398cfc7d795642642bde6a262384a0b3 2024-12-04T20:17:20,886 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-04T20:17:20,886 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-04T20:17:20,886 DEBUG [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-04T20:17:20,886 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-04T20:17:20,886 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-04T20:17:20,886 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-04T20:17:20,886 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-04T20:17:20,886 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-04T20:17:20,886 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-04T20:17:20,890 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/default/testReplayEditsAfterRegionMovedWithMultiCF/398cfc7d795642642bde6a262384a0b3/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-04T20:17:20,891 INFO [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:20,891 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 398cfc7d795642642bde6a262384a0b3: Waiting for close lock at 1733343440885Running coprocessor pre-close hooks at 1733343440885Disabling compacts and flushes for region at 1733343440885Disabling writes for close at 1733343440885Writing region close event to WAL at 1733343440886 (+1 ms)Running coprocessor post-close hooks at 1733343440890 (+4 ms)Closed at 1733343440890 2024-12-04T20:17:20,891 DEBUG [RS_CLOSE_REGION-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3. 2024-12-04T20:17:20,911 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:17:20,912 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/info/d90b412161474309ba1c5c95dc0e640c is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733343417192.398cfc7d795642642bde6a262384a0b3./info:regioninfo/1733343420260/Put/seqid=0 2024-12-04T20:17:20,918 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741938_1118 (size=8243) 2024-12-04T20:17:20,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741938_1118 (size=8243) 2024-12-04T20:17:20,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741938_1118 (size=8243) 2024-12-04T20:17:20,919 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/info/d90b412161474309ba1c5c95dc0e640c 2024-12-04T20:17:20,926 INFO [regionserver/c2ef38372881:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:17:20,939 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/ns/08bfa6fa02e940d69f790de1885540d1 is 43, key is default/ns:d/1733343404514/Put/seqid=0 2024-12-04T20:17:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741939_1119 (size=5153) 2024-12-04T20:17:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741939_1119 (size=5153) 2024-12-04T20:17:20,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741939_1119 (size=5153) 2024-12-04T20:17:20,946 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/ns/08bfa6fa02e940d69f790de1885540d1 2024-12-04T20:17:20,966 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/table/a15985b750664c858122a362e5dd9ee1 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733343417621/Put/seqid=0 2024-12-04T20:17:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741940_1120 (size=5431) 2024-12-04T20:17:20,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741940_1120 (size=5431) 2024-12-04T20:17:20,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741940_1120 (size=5431) 2024-12-04T20:17:20,972 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/table/a15985b750664c858122a362e5dd9ee1 2024-12-04T20:17:20,978 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/info/d90b412161474309ba1c5c95dc0e640c as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/info/d90b412161474309ba1c5c95dc0e640c 2024-12-04T20:17:20,983 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/info/d90b412161474309ba1c5c95dc0e640c, entries=18, sequenceid=21, filesize=8.0 K 2024-12-04T20:17:20,985 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/ns/08bfa6fa02e940d69f790de1885540d1 as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/ns/08bfa6fa02e940d69f790de1885540d1 2024-12-04T20:17:20,990 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/ns/08bfa6fa02e940d69f790de1885540d1, entries=2, sequenceid=21, filesize=5.0 K 2024-12-04T20:17:20,991 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/.tmp/table/a15985b750664c858122a362e5dd9ee1 as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/table/a15985b750664c858122a362e5dd9ee1 2024-12-04T20:17:20,995 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/table/a15985b750664c858122a362e5dd9ee1, entries=2, sequenceid=21, filesize=5.3 K 2024-12-04T20:17:20,996 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false 2024-12-04T20:17:21,000 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-04T20:17:21,001 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-04T20:17:21,001 INFO [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-04T20:17:21,001 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733343440886Running coprocessor pre-close hooks at 1733343440886Disabling compacts and flushes for region at 1733343440886Disabling writes for close at 1733343440886Obtaining lock to block concurrent updates at 1733343440886Preparing flush snapshotting stores in 1588230740 at 1733343440886Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733343440887 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733343440887Flushing 1588230740/info: creating writer at 1733343440887Flushing 1588230740/info: appending metadata at 1733343440911 (+24 ms)Flushing 1588230740/info: closing flushed file at 1733343440911Flushing 1588230740/ns: creating writer at 1733343440925 (+14 ms)Flushing 1588230740/ns: appending metadata at 1733343440939 (+14 ms)Flushing 1588230740/ns: closing flushed file at 1733343440939Flushing 1588230740/table: creating writer at 1733343440952 (+13 ms)Flushing 1588230740/table: appending metadata at 1733343440965 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733343440965Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b655967: reopening flushed file at 1733343440977 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@669169a1: reopening flushed file at 1733343440984 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@564b4e4b: reopening flushed file at 1733343440990 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 110ms, sequenceid=21, compaction requested=false at 1733343440996 (+6 ms)Writing region close event to WAL at 1733343440997 (+1 ms)Running coprocessor post-close hooks at 1733343441001 (+4 ms)Closed at 1733343441001 2024-12-04T20:17:21,001 DEBUG [RS_CLOSE_META-regionserver/c2ef38372881:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-04T20:17:21,086 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,45313,1733343402534; all regions closed. 2024-12-04T20:17:21,086 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(976): stopping server c2ef38372881,40377,1733343402660; all regions closed. 2024-12-04T20:17:21,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741836_1012 (size=9487) 2024-12-04T20:17:21,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741834_1010 (size=3621) 2024-12-04T20:17:21,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741836_1012 (size=9487) 2024-12-04T20:17:21,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741834_1010 (size=3621) 2024-12-04T20:17:21,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741834_1010 (size=3621) 2024-12-04T20:17:21,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741836_1012 (size=9487) 2024-12-04T20:17:21,091 DEBUG [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs 2024-12-04T20:17:21,091 DEBUG [RS:0;c2ef38372881:45313 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs 2024-12-04T20:17:21,091 INFO [RS:0;c2ef38372881:45313 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2ef38372881%2C45313%2C1733343402534:(num 1733343403847) 2024-12-04T20:17:21,091 INFO [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2ef38372881%2C40377%2C1733343402660.meta:.meta(num 1733343404339) 2024-12-04T20:17:21,091 DEBUG [RS:0;c2ef38372881:45313 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:21,091 INFO [RS:0;c2ef38372881:45313 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:17:21,091 INFO [RS:0;c2ef38372881:45313 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-04T20:17:21,092 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:17:21,092 INFO [RS:0;c2ef38372881:45313 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45313 2024-12-04T20:17:21,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:17:21,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741835_1011 (size=175) 2024-12-04T20:17:21,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741835_1011 (size=175) 2024-12-04T20:17:21,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,45313,1733343402534 2024-12-04T20:17:21,094 INFO [RS:0;c2ef38372881:45313 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:17:21,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741835_1011 (size=175) 2024-12-04T20:17:21,095 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,45313,1733343402534] 2024-12-04T20:17:21,096 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,45313,1733343402534 already deleted, retry=false 2024-12-04T20:17:21,096 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,45313,1733343402534 expired; onlineServers=1 2024-12-04T20:17:21,096 DEBUG [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/oldWALs 2024-12-04T20:17:21,096 INFO [RS:2;c2ef38372881:40377 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL c2ef38372881%2C40377%2C1733343402660:(num 1733343403891) 2024-12-04T20:17:21,096 DEBUG [RS:2;c2ef38372881:40377 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-04T20:17:21,096 INFO [RS:2;c2ef38372881:40377 {}] regionserver.LeaseManager(133): Closed leases 2024-12-04T20:17:21,097 INFO [RS:2;c2ef38372881:40377 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:17:21,097 INFO [RS:2;c2ef38372881:40377 {}] hbase.ChoreService(370): Chore service for: regionserver/c2ef38372881:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-04T20:17:21,097 INFO [RS:2;c2ef38372881:40377 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:17:21,097 INFO [regionserver/c2ef38372881:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:17:21,097 INFO [RS:2;c2ef38372881:40377 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40377 2024-12-04T20:17:21,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/c2ef38372881,40377,1733343402660 2024-12-04T20:17:21,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-04T20:17:21,098 INFO [RS:2;c2ef38372881:40377 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:17:21,099 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [c2ef38372881,40377,1733343402660] 2024-12-04T20:17:21,100 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/c2ef38372881,40377,1733343402660 already deleted, retry=false 2024-12-04T20:17:21,100 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; c2ef38372881,40377,1733343402660 expired; onlineServers=0 2024-12-04T20:17:21,100 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'c2ef38372881,37947,1733343401962' ***** 2024-12-04T20:17:21,100 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-04T20:17:21,100 INFO [M:0;c2ef38372881:37947 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-04T20:17:21,100 INFO [M:0;c2ef38372881:37947 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-04T20:17:21,100 DEBUG [M:0;c2ef38372881:37947 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-04T20:17:21,100 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-04T20:17:21,100 DEBUG [M:0;c2ef38372881:37947 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-04T20:17:21,100 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343403571 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.large.0-1733343403571,5,FailOnTimeoutGroup] 2024-12-04T20:17:21,101 DEBUG [master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343403571 {}] cleaner.HFileCleaner(306): Exit Thread[master/c2ef38372881:0:becomeActiveMaster-HFileCleaner.small.0-1733343403571,5,FailOnTimeoutGroup] 2024-12-04T20:17:21,101 INFO [M:0;c2ef38372881:37947 {}] hbase.ChoreService(370): Chore service for: master/c2ef38372881:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-04T20:17:21,101 INFO [M:0;c2ef38372881:37947 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-04T20:17:21,101 DEBUG [M:0;c2ef38372881:37947 {}] master.HMaster(1795): Stopping service threads 2024-12-04T20:17:21,101 INFO [M:0;c2ef38372881:37947 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-04T20:17:21,101 INFO [M:0;c2ef38372881:37947 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-04T20:17:21,101 INFO [M:0;c2ef38372881:37947 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-04T20:17:21,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-04T20:17:21,102 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-04T20:17:21,102 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-04T20:17:21,102 DEBUG [M:0;c2ef38372881:37947 {}] zookeeper.ZKUtil(347): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-04T20:17:21,102 WARN [M:0;c2ef38372881:37947 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-04T20:17:21,103 INFO [M:0;c2ef38372881:37947 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/.lastflushedseqids 2024-12-04T20:17:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741941_1121 (size=138) 2024-12-04T20:17:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741941_1121 (size=138) 2024-12-04T20:17:21,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741941_1121 (size=138) 2024-12-04T20:17:21,114 INFO [M:0;c2ef38372881:37947 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-04T20:17:21,114 INFO [M:0;c2ef38372881:37947 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-04T20:17:21,114 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-04T20:17:21,114 INFO [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:17:21,114 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:17:21,114 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-04T20:17:21,114 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:17:21,114 INFO [M:0;c2ef38372881:37947 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-04T20:17:21,127 DEBUG [M:0;c2ef38372881:37947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9ecc3f928df4e43a8b4f4e91be14dba is 82, key is hbase:meta,,1/info:regioninfo/1733343404445/Put/seqid=0 2024-12-04T20:17:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741942_1122 (size=5672) 2024-12-04T20:17:21,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741942_1122 (size=5672) 2024-12-04T20:17:21,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741942_1122 (size=5672) 2024-12-04T20:17:21,133 INFO [M:0;c2ef38372881:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9ecc3f928df4e43a8b4f4e91be14dba 2024-12-04T20:17:21,151 DEBUG [M:0;c2ef38372881:37947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffd2543bb47345c591f7c1162d4f04bc is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733343417627/Put/seqid=0 2024-12-04T20:17:21,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741943_1123 (size=7756) 2024-12-04T20:17:21,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741943_1123 (size=7756) 2024-12-04T20:17:21,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741943_1123 (size=7756) 2024-12-04T20:17:21,158 INFO [M:0;c2ef38372881:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffd2543bb47345c591f7c1162d4f04bc 2024-12-04T20:17:21,163 INFO [M:0;c2ef38372881:37947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffd2543bb47345c591f7c1162d4f04bc 2024-12-04T20:17:21,177 DEBUG [M:0;c2ef38372881:37947 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de22a4bdcbe3420db642199954d306c2 is 69, key is c2ef38372881,40377,1733343402660/rs:state/1733343403727/Put/seqid=0 2024-12-04T20:17:21,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741944_1124 (size=5440) 2024-12-04T20:17:21,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741944_1124 (size=5440) 2024-12-04T20:17:21,183 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741944_1124 (size=5440) 2024-12-04T20:17:21,183 INFO [M:0;c2ef38372881:37947 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de22a4bdcbe3420db642199954d306c2 2024-12-04T20:17:21,188 INFO [M:0;c2ef38372881:37947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for de22a4bdcbe3420db642199954d306c2 2024-12-04T20:17:21,189 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/e9ecc3f928df4e43a8b4f4e91be14dba as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e9ecc3f928df4e43a8b4f4e91be14dba 2024-12-04T20:17:21,194 INFO [M:0;c2ef38372881:37947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/e9ecc3f928df4e43a8b4f4e91be14dba, entries=8, sequenceid=168, filesize=5.5 K 2024-12-04T20:17:21,195 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ffd2543bb47345c591f7c1162d4f04bc as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffd2543bb47345c591f7c1162d4f04bc 2024-12-04T20:17:21,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,195 INFO [RS:0;c2ef38372881:45313 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:17:21,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45313-0x100a6e0b14d0001, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,195 INFO [RS:0;c2ef38372881:45313 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,45313,1733343402534; zookeeper connection closed. 2024-12-04T20:17:21,196 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@37fe7068 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@37fe7068 2024-12-04T20:17:21,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,199 INFO [RS:2;c2ef38372881:40377 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:17:21,199 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40377-0x100a6e0b14d0003, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,199 INFO [RS:2;c2ef38372881:40377 {}] regionserver.HRegionServer(1031): Exiting; stopping=c2ef38372881,40377,1733343402660; zookeeper connection closed. 2024-12-04T20:17:21,199 INFO [M:0;c2ef38372881:37947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ffd2543bb47345c591f7c1162d4f04bc 2024-12-04T20:17:21,200 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@4808a8f6 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@4808a8f6 2024-12-04T20:17:21,200 INFO [M:0;c2ef38372881:37947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ffd2543bb47345c591f7c1162d4f04bc, entries=17, sequenceid=168, filesize=7.6 K 2024-12-04T20:17:21,200 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-04T20:17:21,201 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/de22a4bdcbe3420db642199954d306c2 as hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de22a4bdcbe3420db642199954d306c2 2024-12-04T20:17:21,205 INFO [M:0;c2ef38372881:37947 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for de22a4bdcbe3420db642199954d306c2 2024-12-04T20:17:21,205 INFO [M:0;c2ef38372881:37947 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:44433/user/jenkins/test-data/f037afc2-d638-d6b4-d544-3824947db061/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/de22a4bdcbe3420db642199954d306c2, entries=3, sequenceid=168, filesize=5.3 K 2024-12-04T20:17:21,207 INFO [M:0;c2ef38372881:37947 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false 2024-12-04T20:17:21,208 INFO [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-04T20:17:21,208 DEBUG [M:0;c2ef38372881:37947 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733343441114Disabling compacts and flushes for region at 1733343441114Disabling writes for close at 1733343441114Obtaining lock to block concurrent updates at 1733343441114Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733343441114Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733343441114Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733343441115 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733343441115Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733343441127 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733343441127Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733343441136 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733343441151 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733343441151Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733343441163 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733343441176 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733343441176Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5f869e26: reopening flushed file at 1733343441188 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@291c2d74: reopening flushed file at 1733343441194 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@19c83a38: reopening flushed file at 1733343441200 (+6 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false at 1733343441207 (+7 ms)Writing region close event to WAL at 1733343441208 (+1 ms)Closed at 1733343441208 2024-12-04T20:17:21,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38071 is added to blk_1073741830_1006 (size=83618) 2024-12-04T20:17:21,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37737 is added to blk_1073741830_1006 (size=83618) 2024-12-04T20:17:21,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42009 is added to blk_1073741830_1006 (size=83618) 2024-12-04T20:17:21,211 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-04T20:17:21,211 INFO [M:0;c2ef38372881:37947 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-04T20:17:21,212 INFO [M:0;c2ef38372881:37947 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:37947 2024-12-04T20:17:21,212 INFO [M:0;c2ef38372881:37947 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-04T20:17:21,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,313 INFO [M:0;c2ef38372881:37947 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-04T20:17:21,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37947-0x100a6e0b14d0000, quorum=127.0.0.1:55668, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-04T20:17:21,319 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420992 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420992 (inode 16655) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733343420634/wal.1733343420992 (inode 16655) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T20:17:21,320 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733343412709/wal.1733343412820 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:21,320 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733343436111/wal.1733343436375 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:21,322 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343430487 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343430487 (inode 16678) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733343421163/wal.1733343430487 (inode 16678) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T20:17:21,323 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733343405928/wal.1733343406018 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:21,325 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343431378 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343431378 (inode 16704) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733343430748/wal.1733343431378 (inode 16704) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T20:17:21,326 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733343431570/wal.1733343431655 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-04T20:17:21,328 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal.1733343405764 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal.1733343405764 (inode 16485) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733343405427/wal.1733343405764 (inode 16485) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T20:17:21,331 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal.1733343405170 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal.1733343405170 (inode 16462) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733343404784/wal.1733343405170 (inode 16462) Holder DFSClient_NONMAPREDUCE_1811373311_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy45.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy46.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-04T20:17:21,342 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:17:21,344 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:17:21,344 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:17:21,344 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:17:21,345 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,STOPPED} 2024-12-04T20:17:21,347 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:17:21,347 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:17:21,347 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-700201894-172.17.0.2-1733343399373 (Datanode Uuid 28e2ccbb-c2af-4818-8139-7a6bd3c977ec) service to localhost/127.0.0.1:44433 2024-12-04T20:17:21,347 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:17:21,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data5/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,348 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data6/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,349 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:17:21,355 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:17:21,355 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:17:21,355 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:17:21,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:17:21,356 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,STOPPED} 2024-12-04T20:17:21,357 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:17:21,357 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:17:21,357 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:17:21,357 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-700201894-172.17.0.2-1733343399373 (Datanode Uuid cbc111ee-df59-4e3c-8cc5-988e06dc269b) service to localhost/127.0.0.1:44433 2024-12-04T20:17:21,358 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data3/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,358 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data4/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,358 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:17:21,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-04T20:17:21,365 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:17:21,365 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:17:21,365 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:17:21,366 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,STOPPED} 2024-12-04T20:17:21,366 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-04T20:17:21,367 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-04T20:17:21,367 WARN [BP-700201894-172.17.0.2-1733343399373 heartbeating to localhost/127.0.0.1:44433 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-700201894-172.17.0.2-1733343399373 (Datanode Uuid 65fdf03d-c3df-49c9-aab8-3a2af9efceef) service to localhost/127.0.0.1:44433 2024-12-04T20:17:21,367 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-04T20:17:21,367 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data1/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,367 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/cluster_d100bacb-207e-c72a-c8df-96c6b31b3c65/data/data2/current/BP-700201894-172.17.0.2-1733343399373 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-04T20:17:21,367 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-04T20:17:21,375 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-04T20:17:21,376 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-04T20:17:21,376 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-04T20:17:21,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-04T20:17:21,376 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/d7643b34-6741-cddd-20eb-90e21bc4eed2/hadoop.log.dir/,STOPPED} 2024-12-04T20:17:21,386 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-04T20:17:21,434 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down