2024-12-03 09:25:07,680 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 09:25:07,690 main DEBUG Took 0.008535 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 09:25:07,690 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 09:25:07,690 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 09:25:07,691 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 09:25:07,692 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,698 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 09:25:07,709 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,710 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,711 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,711 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,712 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,712 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,713 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,713 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,713 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,714 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,714 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,715 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,715 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,716 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,716 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,716 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,717 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,717 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,717 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,718 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,718 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 09:25:07,719 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,719 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 09:25:07,720 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 09:25:07,721 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 09:25:07,723 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 09:25:07,723 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 09:25:07,724 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 09:25:07,725 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 09:25:07,731 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 09:25:07,734 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 09:25:07,735 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 09:25:07,736 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 09:25:07,736 main DEBUG createAppenders(={Console}) 2024-12-03 09:25:07,737 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 09:25:07,737 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 09:25:07,737 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 09:25:07,737 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 09:25:07,738 main DEBUG OutputStream closed 2024-12-03 09:25:07,738 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 09:25:07,738 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 09:25:07,738 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 09:25:07,799 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 09:25:07,801 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 09:25:07,802 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 09:25:07,803 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 09:25:07,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 09:25:07,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 09:25:07,804 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 09:25:07,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 09:25:07,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 09:25:07,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 09:25:07,805 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 09:25:07,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 09:25:07,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 09:25:07,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 09:25:07,806 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 09:25:07,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 09:25:07,807 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 09:25:07,808 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 09:25:07,810 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 09:25:07,810 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 09:25:07,811 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 09:25:07,811 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T09:25:08,048 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06 2024-12-03 09:25:08,051 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 09:25:08,051 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T09:25:08,059 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T09:25:08,065 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplayValueCompression timeout: 13 mins 2024-12-03T09:25:08,082 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T09:25:08,121 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T09:25:08,121 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T09:25:08,132 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T09:25:08,144 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330, deleteOnExit=true 2024-12-03T09:25:08,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T09:25:08,145 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/test.cache.data in system properties and HBase conf 2024-12-03T09:25:08,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T09:25:08,146 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir in system properties and HBase conf 2024-12-03T09:25:08,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T09:25:08,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T09:25:08,147 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T09:25:08,220 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T09:25:08,300 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T09:25:08,303 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T09:25:08,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T09:25:08,304 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T09:25:08,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T09:25:08,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T09:25:08,305 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T09:25:08,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T09:25:08,306 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T09:25:08,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T09:25:08,307 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/nfs.dump.dir in system properties and HBase conf 2024-12-03T09:25:08,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/java.io.tmpdir in system properties and HBase conf 2024-12-03T09:25:08,308 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T09:25:08,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T09:25:08,309 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T09:25:09,355 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T09:25:09,415 INFO [Time-limited test {}] log.Log(170): Logging initialized @2336ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T09:25:09,476 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:25:09,526 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:25:09,544 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:25:09,544 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:25:09,545 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T09:25:09,557 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:25:09,560 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:25:09,561 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:25:09,733 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2c8b857b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/java.io.tmpdir/jetty-localhost-33757-hadoop-hdfs-3_4_1-tests_jar-_-any-12747649592752172710/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T09:25:09,738 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:33757} 2024-12-03T09:25:09,739 INFO [Time-limited test {}] server.Server(415): Started @2660ms 2024-12-03T09:25:10,241 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:25:10,249 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:25:10,250 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:25:10,250 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:25:10,250 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T09:25:10,251 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:25:10,252 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:25:10,361 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@17d1a8d5{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/java.io.tmpdir/jetty-localhost-39651-hadoop-hdfs-3_4_1-tests_jar-_-any-3373240601252775549/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:10,362 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:39651} 2024-12-03T09:25:10,362 INFO [Time-limited test {}] server.Server(415): Started @3283ms 2024-12-03T09:25:10,412 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T09:25:10,511 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:25:10,515 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:25:10,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:25:10,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:25:10,517 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T09:25:10,520 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:25:10,521 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:25:10,619 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4464725f{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/java.io.tmpdir/jetty-localhost-45765-hadoop-hdfs-3_4_1-tests_jar-_-any-12009105731123868028/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:10,619 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:45765} 2024-12-03T09:25:10,619 INFO [Time-limited test {}] server.Server(415): Started @3541ms 2024-12-03T09:25:10,621 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T09:25:10,652 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T09:25:10,656 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T09:25:10,657 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T09:25:10,657 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T09:25:10,657 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T09:25:10,658 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,AVAILABLE} 2024-12-03T09:25:10,659 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T09:25:10,751 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@26f8cb79{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/java.io.tmpdir/jetty-localhost-34949-hadoop-hdfs-3_4_1-tests_jar-_-any-13891977666797031946/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:10,751 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:34949} 2024-12-03T09:25:10,751 INFO [Time-limited test {}] server.Server(415): Started @3673ms 2024-12-03T09:25:10,753 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T09:25:12,395 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data3/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,395 WARN [Thread-127 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data4/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,395 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data2/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,395 WARN [Thread-128 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data5/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,395 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data1/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,395 WARN [Thread-129 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data6/current/BP-1349158104-172.17.0.2-1733217908790/current, will proceed with Du for space computation calculation, 2024-12-03T09:25:12,438 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T09:25:12,438 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T09:25:12,438 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T09:25:12,479 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4d6c85d6f4ba57f with lease ID 0xdeed55753959ddca: Processing first storage report for DS-5878274c-1618-4fe8-9285-33433066f5e3 from datanode DatanodeRegistration(127.0.0.1:36933, datanodeUuid=f16e760b-d084-42b1-9f6c-0da663611f76, infoPort=40327, infoSecurePort=0, ipcPort=38187, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,480 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d6c85d6f4ba57f with lease ID 0xdeed55753959ddca: from storage DS-5878274c-1618-4fe8-9285-33433066f5e3 node DatanodeRegistration(127.0.0.1:36933, datanodeUuid=f16e760b-d084-42b1-9f6c-0da663611f76, infoPort=40327, infoSecurePort=0, ipcPort=38187, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,480 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbb067025cd55c3f with lease ID 0xdeed55753959ddc8: Processing first storage report for DS-92f817a3-291f-4319-bace-a448fa4c2594 from datanode DatanodeRegistration(127.0.0.1:34311, datanodeUuid=c42efa65-85a4-4907-ab2f-d747318fb4b5, infoPort=37521, infoSecurePort=0, ipcPort=46303, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbb067025cd55c3f with lease ID 0xdeed55753959ddc8: from storage DS-92f817a3-291f-4319-bace-a448fa4c2594 node DatanodeRegistration(127.0.0.1:34311, datanodeUuid=c42efa65-85a4-4907-ab2f-d747318fb4b5, infoPort=37521, infoSecurePort=0, ipcPort=46303, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x668a6e8afd540152 with lease ID 0xdeed55753959ddc9: Processing first storage report for DS-6cdfbc66-232b-48a4-adbb-355e49d63215 from datanode DatanodeRegistration(127.0.0.1:41153, datanodeUuid=6c8444e9-a104-4ae1-b929-4bed1af47d3d, infoPort=40249, infoSecurePort=0, ipcPort=43269, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x668a6e8afd540152 with lease ID 0xdeed55753959ddc9: from storage DS-6cdfbc66-232b-48a4-adbb-355e49d63215 node DatanodeRegistration(127.0.0.1:41153, datanodeUuid=6c8444e9-a104-4ae1-b929-4bed1af47d3d, infoPort=40249, infoSecurePort=0, ipcPort=43269, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe4d6c85d6f4ba57f with lease ID 0xdeed55753959ddca: Processing first storage report for DS-6323af39-f653-4ab7-8c28-9704a58c2d9f from datanode DatanodeRegistration(127.0.0.1:36933, datanodeUuid=f16e760b-d084-42b1-9f6c-0da663611f76, infoPort=40327, infoSecurePort=0, ipcPort=38187, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe4d6c85d6f4ba57f with lease ID 0xdeed55753959ddca: from storage DS-6323af39-f653-4ab7-8c28-9704a58c2d9f node DatanodeRegistration(127.0.0.1:36933, datanodeUuid=f16e760b-d084-42b1-9f6c-0da663611f76, infoPort=40327, infoSecurePort=0, ipcPort=38187, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,481 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbbb067025cd55c3f with lease ID 0xdeed55753959ddc8: Processing first storage report for DS-3c0b3807-c130-4287-ac6a-752683f772f3 from datanode DatanodeRegistration(127.0.0.1:34311, datanodeUuid=c42efa65-85a4-4907-ab2f-d747318fb4b5, infoPort=37521, infoSecurePort=0, ipcPort=46303, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbbb067025cd55c3f with lease ID 0xdeed55753959ddc8: from storage DS-3c0b3807-c130-4287-ac6a-752683f772f3 node DatanodeRegistration(127.0.0.1:34311, datanodeUuid=c42efa65-85a4-4907-ab2f-d747318fb4b5, infoPort=37521, infoSecurePort=0, ipcPort=46303, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x668a6e8afd540152 with lease ID 0xdeed55753959ddc9: Processing first storage report for DS-aea319bb-7d15-4d40-980e-9b6b1b18b3c6 from datanode DatanodeRegistration(127.0.0.1:41153, datanodeUuid=6c8444e9-a104-4ae1-b929-4bed1af47d3d, infoPort=40249, infoSecurePort=0, ipcPort=43269, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790) 2024-12-03T09:25:12,482 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x668a6e8afd540152 with lease ID 0xdeed55753959ddc9: from storage DS-aea319bb-7d15-4d40-980e-9b6b1b18b3c6 node DatanodeRegistration(127.0.0.1:41153, datanodeUuid=6c8444e9-a104-4ae1-b929-4bed1af47d3d, infoPort=40249, infoSecurePort=0, ipcPort=43269, storageInfo=lv=-57;cid=testClusterID;nsid=675415169;c=1733217908790), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T09:25:12,565 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06 2024-12-03T09:25:12,622 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/zookeeper_0, clientPort=57184, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T09:25:12,630 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=57184 2024-12-03T09:25:12,638 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:12,641 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:12,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741825_1001 (size=7) 2024-12-03T09:25:12,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741825_1001 (size=7) 2024-12-03T09:25:12,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741825_1001 (size=7) 2024-12-03T09:25:13,235 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 with version=8 2024-12-03T09:25:13,235 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/hbase-staging 2024-12-03T09:25:13,510 INFO [Time-limited test {}] client.ConnectionUtils(128): master/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:25:13,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:13,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:13,522 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:25:13,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:13,522 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:25:13,630 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T09:25:13,678 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T09:25:13,685 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T09:25:13,688 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:25:13,709 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 70862 (auto-detected) 2024-12-03T09:25:13,710 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-03T09:25:13,731 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:33521 2024-12-03T09:25:13,750 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:33521 connecting to ZooKeeper ensemble=127.0.0.1:57184 2024-12-03T09:25:13,850 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:335210x0, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:25:13,853 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:33521-0x1019a8282cb0000 connected 2024-12-03T09:25:13,945 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:13,949 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:13,960 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:13,964 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41, hbase.cluster.distributed=false 2024-12-03T09:25:13,983 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:25:13,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=33521 2024-12-03T09:25:13,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=33521 2024-12-03T09:25:13,988 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=33521 2024-12-03T09:25:13,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=33521 2024-12-03T09:25:13,990 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=33521 2024-12-03T09:25:14,086 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:25:14,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,088 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:25:14,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,088 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:25:14,091 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T09:25:14,093 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:25:14,094 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40779 2024-12-03T09:25:14,096 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40779 connecting to ZooKeeper ensemble=127.0.0.1:57184 2024-12-03T09:25:14,097 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,099 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,111 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407790x0, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:25:14,112 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407790x0, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:14,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40779-0x1019a8282cb0001 connected 2024-12-03T09:25:14,117 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T09:25:14,125 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T09:25:14,128 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T09:25:14,134 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:25:14,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40779 2024-12-03T09:25:14,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40779 2024-12-03T09:25:14,135 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40779 2024-12-03T09:25:14,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40779 2024-12-03T09:25:14,136 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40779 2024-12-03T09:25:14,151 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:25:14,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,151 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:25:14,151 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,152 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:25:14,152 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T09:25:14,152 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:25:14,153 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40727 2024-12-03T09:25:14,154 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40727 connecting to ZooKeeper ensemble=127.0.0.1:57184 2024-12-03T09:25:14,155 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,157 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:407270x0, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:25:14,174 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:407270x0, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:14,174 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40727-0x1019a8282cb0002 connected 2024-12-03T09:25:14,175 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T09:25:14,175 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T09:25:14,176 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T09:25:14,178 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:25:14,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40727 2024-12-03T09:25:14,178 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40727 2024-12-03T09:25:14,179 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40727 2024-12-03T09:25:14,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40727 2024-12-03T09:25:14,181 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40727 2024-12-03T09:25:14,196 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/699c9baf3d29:0 server-side Connection retries=45 2024-12-03T09:25:14,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,196 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,196 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T09:25:14,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T09:25:14,197 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T09:25:14,197 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T09:25:14,197 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T09:25:14,198 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46197 2024-12-03T09:25:14,200 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46197 connecting to ZooKeeper ensemble=127.0.0.1:57184 2024-12-03T09:25:14,202 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,205 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,216 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461970x0, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T09:25:14,217 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461970x0, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:14,217 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46197-0x1019a8282cb0003 connected 2024-12-03T09:25:14,217 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T09:25:14,218 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T09:25:14,219 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T09:25:14,221 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T09:25:14,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46197 2024-12-03T09:25:14,222 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46197 2024-12-03T09:25:14,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46197 2024-12-03T09:25:14,223 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46197 2024-12-03T09:25:14,224 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46197 2024-12-03T09:25:14,239 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;699c9baf3d29:33521 2024-12-03T09:25:14,240 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/699c9baf3d29,33521,1733217913369 2024-12-03T09:25:14,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,248 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,250 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/699c9baf3d29,33521,1733217913369 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,279 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,280 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T09:25:14,281 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/699c9baf3d29,33521,1733217913369 from backup master directory 2024-12-03T09:25:14,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/699c9baf3d29,33521,1733217913369 2024-12-03T09:25:14,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,290 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T09:25:14,292 WARN [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:25:14,292 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=699c9baf3d29,33521,1733217913369 2024-12-03T09:25:14,295 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T09:25:14,296 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T09:25:14,348 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/hbase.id] with ID: b9c76278-f69a-4259-b357-0b850c15c1c5 2024-12-03T09:25:14,349 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/.tmp/hbase.id 2024-12-03T09:25:14,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741826_1002 (size=42) 2024-12-03T09:25:14,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741826_1002 (size=42) 2024-12-03T09:25:14,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741826_1002 (size=42) 2024-12-03T09:25:14,364 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/.tmp/hbase.id]:[hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/hbase.id] 2024-12-03T09:25:14,411 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T09:25:14,415 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T09:25:14,431 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 15ms. 2024-12-03T09:25:14,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,437 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741827_1003 (size=196) 2024-12-03T09:25:14,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741827_1003 (size=196) 2024-12-03T09:25:14,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741827_1003 (size=196) 2024-12-03T09:25:14,469 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:25:14,471 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T09:25:14,476 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:14,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741828_1004 (size=1189) 2024-12-03T09:25:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741828_1004 (size=1189) 2024-12-03T09:25:14,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741828_1004 (size=1189) 2024-12-03T09:25:14,524 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store 2024-12-03T09:25:14,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741829_1005 (size=34) 2024-12-03T09:25:14,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741829_1005 (size=34) 2024-12-03T09:25:14,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741829_1005 (size=34) 2024-12-03T09:25:14,551 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T09:25:14,554 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:14,555 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T09:25:14,555 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:14,556 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:14,557 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T09:25:14,557 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:14,557 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:14,558 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733217914555Disabling compacts and flushes for region at 1733217914555Disabling writes for close at 1733217914557 (+2 ms)Writing region close event to WAL at 1733217914557Closed at 1733217914557 2024-12-03T09:25:14,560 WARN [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/.initializing 2024-12-03T09:25:14,560 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/WALs/699c9baf3d29,33521,1733217913369 2024-12-03T09:25:14,568 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:14,582 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C33521%2C1733217913369, suffix=, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/WALs/699c9baf3d29,33521,1733217913369, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/oldWALs, maxLogs=10 2024-12-03T09:25:14,609 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/WALs/699c9baf3d29,33521,1733217913369/699c9baf3d29%2C33521%2C1733217913369.1733217914586, exclude list is [], retry=0 2024-12-03T09:25:14,626 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:14,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:14,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:14,630 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:14,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T09:25:14,668 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/WALs/699c9baf3d29,33521,1733217913369/699c9baf3d29%2C33521%2C1733217913369.1733217914586 2024-12-03T09:25:14,669 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:14,669 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:14,670 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:14,672 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,673 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,703 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,724 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T09:25:14,727 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:14,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:14,729 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,732 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T09:25:14,732 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:14,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:14,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,736 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T09:25:14,736 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:14,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:14,737 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,740 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T09:25:14,740 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:14,741 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:14,741 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,744 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,745 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,750 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,751 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,754 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T09:25:14,758 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T09:25:14,762 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:14,764 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74641495, jitterRate=0.11224494874477386}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T09:25:14,769 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733217914683Initializing all the Stores at 1733217914685 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217914686 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217914686Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217914686Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217914687 (+1 ms)Cleaning up temporary data from old regions at 1733217914751 (+64 ms)Region opened successfully at 1733217914769 (+18 ms) 2024-12-03T09:25:14,770 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T09:25:14,801 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@894442b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:25:14,826 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T09:25:14,835 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T09:25:14,835 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T09:25:14,838 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T09:25:14,839 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T09:25:14,843 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 3 msec 2024-12-03T09:25:14,843 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T09:25:14,863 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T09:25:14,871 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T09:25:14,921 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T09:25:14,924 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T09:25:14,926 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T09:25:14,932 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T09:25:14,934 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T09:25:14,938 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T09:25:14,942 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T09:25:14,943 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T09:25:14,953 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T09:25:14,971 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T09:25:14,983 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T09:25:14,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:14,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:14,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:14,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:14,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:14,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,000 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=699c9baf3d29,33521,1733217913369, sessionid=0x1019a8282cb0000, setting cluster-up flag (Was=false) 2024-12-03T09:25:15,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,027 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,059 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T09:25:15,063 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=699c9baf3d29,33521,1733217913369 2024-12-03T09:25:15,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,090 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,121 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T09:25:15,124 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=699c9baf3d29,33521,1733217913369 2024-12-03T09:25:15,134 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T09:25:15,213 INFO [AsyncFSWAL-0-hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData-prefix:699c9baf3d29,33521,1733217913369 {}] compress.Compression(560): Loaded codec org.apache.hadoop.hbase.io.compress.ReusableStreamGzipCodec for compression algorithm GZ 2024-12-03T09:25:15,226 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T09:25:15,227 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(746): ClusterId : b9c76278-f69a-4259-b357-0b850c15c1c5 2024-12-03T09:25:15,227 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(746): ClusterId : b9c76278-f69a-4259-b357-0b850c15c1c5 2024-12-03T09:25:15,227 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(746): ClusterId : b9c76278-f69a-4259-b357-0b850c15c1c5 2024-12-03T09:25:15,230 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T09:25:15,230 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T09:25:15,230 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T09:25:15,236 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T09:25:15,242 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T09:25:15,250 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T09:25:15,250 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T09:25:15,250 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T09:25:15,250 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T09:25:15,250 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T09:25:15,250 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T09:25:15,248 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 699c9baf3d29,33521,1733217913369 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T09:25:15,270 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T09:25:15,270 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T09:25:15,270 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T09:25:15,271 DEBUG [RS:2;699c9baf3d29:46197 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7023a38e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:25:15,271 DEBUG [RS:1;699c9baf3d29:40727 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a782bc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:25:15,271 DEBUG [RS:0;699c9baf3d29:40779 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ae5be7c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=699c9baf3d29/172.17.0.2:0 2024-12-03T09:25:15,272 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:25:15,272 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:25:15,272 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:25:15,272 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/699c9baf3d29:0, corePoolSize=5, maxPoolSize=5 2024-12-03T09:25:15,272 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/699c9baf3d29:0, corePoolSize=10, maxPoolSize=10 2024-12-03T09:25:15,273 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,273 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:25:15,273 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,282 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:25:15,283 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T09:25:15,287 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;699c9baf3d29:46197 2024-12-03T09:25:15,287 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733217945287 2024-12-03T09:25:15,289 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:15,289 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T09:25:15,290 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;699c9baf3d29:40727 2024-12-03T09:25:15,290 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;699c9baf3d29:40779 2024-12-03T09:25:15,289 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T09:25:15,290 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T09:25:15,292 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T09:25:15,292 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T09:25:15,292 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T09:25:15,292 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T09:25:15,292 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T09:25:15,292 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T09:25:15,292 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T09:25:15,292 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T09:25:15,293 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T09:25:15,294 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T09:25:15,295 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T09:25:15,295 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T09:25:15,295 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T09:25:15,295 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(2659): reportForDuty to master=699c9baf3d29,33521,1733217913369 with port=40727, startcode=1733217914150 2024-12-03T09:25:15,295 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(2659): reportForDuty to master=699c9baf3d29,33521,1733217913369 with port=46197, startcode=1733217914195 2024-12-03T09:25:15,295 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(2659): reportForDuty to master=699c9baf3d29,33521,1733217913369 with port=40779, startcode=1733217914054 2024-12-03T09:25:15,296 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,301 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T09:25:15,302 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T09:25:15,302 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T09:25:15,304 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T09:25:15,304 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T09:25:15,308 DEBUG [RS:0;699c9baf3d29:40779 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T09:25:15,308 DEBUG [RS:1;699c9baf3d29:40727 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T09:25:15,308 DEBUG [RS:2;699c9baf3d29:46197 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T09:25:15,311 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217915306,5,FailOnTimeoutGroup] 2024-12-03T09:25:15,314 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217915312,5,FailOnTimeoutGroup] 2024-12-03T09:25:15,314 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,314 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T09:25:15,330 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,330 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741831_1007 (size=1321) 2024-12-03T09:25:15,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741831_1007 (size=1321) 2024-12-03T09:25:15,334 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741831_1007 (size=1321) 2024-12-03T09:25:15,336 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T09:25:15,337 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 2024-12-03T09:25:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741832_1008 (size=32) 2024-12-03T09:25:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741832_1008 (size=32) 2024-12-03T09:25:15,353 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741832_1008 (size=32) 2024-12-03T09:25:15,355 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58869, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T09:25:15,355 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:59583, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T09:25:15,355 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:15,358 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57235, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T09:25:15,361 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,362 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T09:25:15,363 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(517): Registering regionserver=699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,365 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T09:25:15,365 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:15,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:15,367 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T09:25:15,370 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T09:25:15,370 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:15,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:15,372 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T09:25:15,375 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T09:25:15,375 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,375 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:15,375 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(517): Registering regionserver=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:15,378 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T09:25:15,379 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 2024-12-03T09:25:15,380 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37427 2024-12-03T09:25:15,380 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T09:25:15,381 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,381 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T09:25:15,381 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 2024-12-03T09:25:15,381 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] master.ServerManager(517): Registering regionserver=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,382 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37427 2024-12-03T09:25:15,382 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:15,382 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T09:25:15,383 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:15,383 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T09:25:15,385 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:15,386 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 2024-12-03T09:25:15,386 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:37427 2024-12-03T09:25:15,386 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T09:25:15,386 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:15,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T09:25:15,390 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T09:25:15,391 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T09:25:15,394 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T09:25:15,398 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:15,400 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72603679, jitterRate=0.08187912404537201}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T09:25:15,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733217915356Initializing all the Stores at 1733217915358 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217915358Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217915361 (+3 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217915362 (+1 ms)Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217915362Cleaning up temporary data from old regions at 1733217915390 (+28 ms)Region opened successfully at 1733217915403 (+13 ms) 2024-12-03T09:25:15,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T09:25:15,403 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T09:25:15,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T09:25:15,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T09:25:15,403 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T09:25:15,404 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T09:25:15,405 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733217915403Disabling compacts and flushes for region at 1733217915403Disabling writes for close at 1733217915403Writing region close event to WAL at 1733217915404 (+1 ms)Closed at 1733217915404 2024-12-03T09:25:15,408 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:25:15,408 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T09:25:15,413 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T09:25:15,422 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:25:15,425 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:25:15,425 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T09:25:15,448 DEBUG [RS:1;699c9baf3d29:40727 {}] zookeeper.ZKUtil(111): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,448 DEBUG [RS:2;699c9baf3d29:46197 {}] zookeeper.ZKUtil(111): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,448 WARN [RS:1;699c9baf3d29:40727 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:25:15,448 WARN [RS:2;699c9baf3d29:46197 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:25:15,448 INFO [RS:2;699c9baf3d29:46197 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:15,448 INFO [RS:1;699c9baf3d29:40727 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:15,448 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,448 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,448 DEBUG [RS:0;699c9baf3d29:40779 {}] zookeeper.ZKUtil(111): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,448 WARN [RS:0;699c9baf3d29:40779 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T09:25:15,449 INFO [RS:0;699c9baf3d29:40779 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:15,449 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,449 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [699c9baf3d29,46197,1733217914195] 2024-12-03T09:25:15,449 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [699c9baf3d29,40779,1733217914054] 2024-12-03T09:25:15,449 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [699c9baf3d29,40727,1733217914150] 2024-12-03T09:25:15,472 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T09:25:15,472 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T09:25:15,472 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T09:25:15,484 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T09:25:15,484 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T09:25:15,484 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T09:25:15,490 INFO [RS:2;699c9baf3d29:46197 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:25:15,490 INFO [RS:0;699c9baf3d29:40779 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:25:15,490 INFO [RS:1;699c9baf3d29:40727 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:25:15,490 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,490 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,490 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,491 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T09:25:15,492 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T09:25:15,492 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T09:25:15,496 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T09:25:15,496 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T09:25:15,498 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T09:25:15,498 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,498 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,498 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:25:15,499 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:25:15,499 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0, corePoolSize=2, maxPoolSize=2 2024-12-03T09:25:15,499 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/699c9baf3d29:0, corePoolSize=1, maxPoolSize=1 2024-12-03T09:25:15,500 DEBUG [RS:1;699c9baf3d29:40727 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,500 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,500 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,501 DEBUG [RS:0;699c9baf3d29:40779 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,501 DEBUG [RS:2;699c9baf3d29:46197 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/699c9baf3d29:0, corePoolSize=3, maxPoolSize=3 2024-12-03T09:25:15,508 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,508 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,508 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,508 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,508 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,40727,1733217914150-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,509 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,40779,1733217914054-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:25:15,512 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,512 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,513 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,513 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,513 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,513 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,46197,1733217914195-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:25:15,535 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T09:25:15,535 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T09:25:15,535 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T09:25:15,537 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,40779,1733217914054-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,537 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,46197,1733217914195-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,537 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,40727,1733217914150-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,538 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,538 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,538 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,538 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.Replication(171): 699c9baf3d29,40727,1733217914150 started 2024-12-03T09:25:15,538 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.Replication(171): 699c9baf3d29,46197,1733217914195 started 2024-12-03T09:25:15,538 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.Replication(171): 699c9baf3d29,40779,1733217914054 started 2024-12-03T09:25:15,563 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,563 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,563 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:15,563 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1482): Serving as 699c9baf3d29,40727,1733217914150, RpcServer on 699c9baf3d29/172.17.0.2:40727, sessionid=0x1019a8282cb0002 2024-12-03T09:25:15,563 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1482): Serving as 699c9baf3d29,46197,1733217914195, RpcServer on 699c9baf3d29/172.17.0.2:46197, sessionid=0x1019a8282cb0003 2024-12-03T09:25:15,563 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1482): Serving as 699c9baf3d29,40779,1733217914054, RpcServer on 699c9baf3d29/172.17.0.2:40779, sessionid=0x1019a8282cb0001 2024-12-03T09:25:15,564 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T09:25:15,564 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T09:25:15,564 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T09:25:15,565 DEBUG [RS:0;699c9baf3d29:40779 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,565 DEBUG [RS:1;699c9baf3d29:40727 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,565 DEBUG [RS:2;699c9baf3d29:46197 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,565 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,40727,1733217914150' 2024-12-03T09:25:15,565 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,46197,1733217914195' 2024-12-03T09:25:15,565 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,40779,1733217914054' 2024-12-03T09:25:15,565 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T09:25:15,565 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T09:25:15,565 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T09:25:15,566 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T09:25:15,566 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T09:25:15,566 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T09:25:15,567 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T09:25:15,567 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T09:25:15,567 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T09:25:15,567 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T09:25:15,567 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T09:25:15,567 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T09:25:15,567 DEBUG [RS:0;699c9baf3d29:40779 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,567 DEBUG [RS:1;699c9baf3d29:40727 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 699c9baf3d29,40727,1733217914150 2024-12-03T09:25:15,567 DEBUG [RS:2;699c9baf3d29:46197 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 699c9baf3d29,46197,1733217914195 2024-12-03T09:25:15,567 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,40727,1733217914150' 2024-12-03T09:25:15,567 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,40779,1733217914054' 2024-12-03T09:25:15,567 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '699c9baf3d29,46197,1733217914195' 2024-12-03T09:25:15,567 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T09:25:15,567 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T09:25:15,567 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T09:25:15,568 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T09:25:15,568 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T09:25:15,568 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T09:25:15,568 DEBUG [RS:1;699c9baf3d29:40727 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T09:25:15,568 DEBUG [RS:2;699c9baf3d29:46197 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T09:25:15,569 DEBUG [RS:0;699c9baf3d29:40779 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T09:25:15,569 INFO [RS:1;699c9baf3d29:40727 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T09:25:15,569 INFO [RS:2;699c9baf3d29:46197 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T09:25:15,569 INFO [RS:0;699c9baf3d29:40779 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T09:25:15,569 INFO [RS:1;699c9baf3d29:40727 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T09:25:15,569 INFO [RS:2;699c9baf3d29:46197 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T09:25:15,569 INFO [RS:0;699c9baf3d29:40779 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T09:25:15,576 WARN [699c9baf3d29:33521 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T09:25:15,675 INFO [RS:2;699c9baf3d29:46197 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:15,675 INFO [RS:0;699c9baf3d29:40779 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:15,675 INFO [RS:1;699c9baf3d29:40727 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:15,678 INFO [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C40727%2C1733217914150, suffix=, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs, maxLogs=32 2024-12-03T09:25:15,678 INFO [RS:0;699c9baf3d29:40779 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C40779%2C1733217914054, suffix=, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs, maxLogs=32 2024-12-03T09:25:15,678 INFO [RS:2;699c9baf3d29:46197 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C46197%2C1733217914195, suffix=, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,46197,1733217914195, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs, maxLogs=32 2024-12-03T09:25:15,695 DEBUG [RS:1;699c9baf3d29:40727 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150/699c9baf3d29%2C40727%2C1733217914150.1733217915683, exclude list is [], retry=0 2024-12-03T09:25:15,695 DEBUG [RS:0;699c9baf3d29:40779 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054/699c9baf3d29%2C40779%2C1733217914054.1733217915683, exclude list is [], retry=0 2024-12-03T09:25:15,696 DEBUG [RS:2;699c9baf3d29:46197 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,46197,1733217914195/699c9baf3d29%2C46197%2C1733217914195.1733217915683, exclude list is [], retry=0 2024-12-03T09:25:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:15,700 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:15,701 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:15,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:15,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:15,702 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:15,706 INFO [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150/699c9baf3d29%2C40727%2C1733217914150.1733217915683 2024-12-03T09:25:15,707 INFO [RS:0;699c9baf3d29:40779 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054/699c9baf3d29%2C40779%2C1733217914054.1733217915683 2024-12-03T09:25:15,707 DEBUG [RS:0;699c9baf3d29:40779 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:15,707 DEBUG [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:15,709 INFO [RS:2;699c9baf3d29:46197 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,46197,1733217914195/699c9baf3d29%2C46197%2C1733217914195.1733217915683 2024-12-03T09:25:15,709 DEBUG [RS:2;699c9baf3d29:46197 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:15,831 DEBUG [699c9baf3d29:33521 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T09:25:15,837 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(204): Hosts are {699c9baf3d29=0} racks are {/default-rack=0} 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T09:25:15,843 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T09:25:15,843 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T09:25:15,843 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T09:25:15,843 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T09:25:15,849 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:15,853 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,40779,1733217914054, state=OPENING 2024-12-03T09:25:15,921 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T09:25:15,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,933 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:15,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:15,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:15,935 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:15,936 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:15,938 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:25:15,940 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,40779,1733217914054}] 2024-12-03T09:25:16,116 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:25:16,118 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:50515, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:25:16,131 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T09:25:16,131 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:16,132 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T09:25:16,135 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C40779%2C1733217914054.meta, suffix=.meta, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs, maxLogs=32 2024-12-03T09:25:16,150 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, exclude list is [], retry=0 2024-12-03T09:25:16,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:16,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:16,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:16,157 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta 2024-12-03T09:25:16,157 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:16,158 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:16,159 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T09:25:16,161 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T09:25:16,165 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T09:25:16,169 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T09:25:16,170 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:16,170 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T09:25:16,170 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T09:25:16,174 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T09:25:16,176 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T09:25:16,176 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:16,177 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T09:25:16,178 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T09:25:16,178 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:16,179 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T09:25:16,181 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T09:25:16,181 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,181 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:16,182 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T09:25:16,183 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T09:25:16,183 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,184 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:16,184 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T09:25:16,186 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:16,188 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:16,191 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T09:25:16,191 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T09:25:16,192 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T09:25:16,194 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T09:25:16,195 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67989756, jitterRate=0.01312631368637085}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T09:25:16,196 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T09:25:16,196 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733217916171Writing region info on filesystem at 1733217916171Initializing all the Stores at 1733217916173 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217916173Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217916174 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217916174Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217916174Cleaning up temporary data from old regions at 1733217916191 (+17 ms)Running coprocessor post-open hooks at 1733217916196 (+5 ms)Region opened successfully at 1733217916196 2024-12-03T09:25:16,202 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733217916107 2024-12-03T09:25:16,212 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T09:25:16,213 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T09:25:16,214 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:16,216 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,40779,1733217914054, state=OPEN 2024-12-03T09:25:16,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:16,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:16,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:16,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:16,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:16,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:16,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:16,227 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:16,227 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:16,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T09:25:16,234 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,40779,1733217914054 in 287 msec 2024-12-03T09:25:16,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T09:25:16,241 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 823 msec 2024-12-03T09:25:16,242 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T09:25:16,242 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T09:25:16,261 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:16,262 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1] 2024-12-03T09:25:16,279 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:25:16,280 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:49573, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:25:16,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1330 sec 2024-12-03T09:25:16,298 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733217916298, completionTime=-1 2024-12-03T09:25:16,301 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T09:25:16,301 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T09:25:16,326 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T09:25:16,326 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733217976326 2024-12-03T09:25:16,326 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733218036326 2024-12-03T09:25:16,326 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 24 msec 2024-12-03T09:25:16,327 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T09:25:16,333 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,334 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,334 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,335 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-699c9baf3d29:33521, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,336 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,336 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,343 DEBUG [master/699c9baf3d29:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T09:25:16,364 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.071sec 2024-12-03T09:25:16,365 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T09:25:16,366 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T09:25:16,367 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T09:25:16,367 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T09:25:16,368 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T09:25:16,368 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T09:25:16,369 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T09:25:16,372 DEBUG [master/699c9baf3d29:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T09:25:16,373 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T09:25:16,374 INFO [master/699c9baf3d29:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=699c9baf3d29,33521,1733217913369-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:16,441 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20c20101, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:25:16,442 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 699c9baf3d29,33521,-1 for getting cluster id 2024-12-03T09:25:16,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T09:25:16,453 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'b9c76278-f69a-4259-b357-0b850c15c1c5' 2024-12-03T09:25:16,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T09:25:16,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "b9c76278-f69a-4259-b357-0b850c15c1c5" 2024-12-03T09:25:16,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64d15519, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:25:16,455 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [699c9baf3d29,33521,-1] 2024-12-03T09:25:16,458 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T09:25:16,461 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:16,461 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T09:25:16,464 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15ed661d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T09:25:16,464 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:16,471 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1] 2024-12-03T09:25:16,471 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:25:16,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45858, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:25:16,492 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=699c9baf3d29,33521,1733217913369 2024-12-03T09:25:16,493 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:37427/hbase 2024-12-03T09:25:16,505 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=369, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=13601 2024-12-03T09:25:16,521 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:16,524 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:16,525 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:16,529 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-20829414, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-20829414, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:16,546 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-20829414/hregion-20829414.1733217916531, exclude list is [], retry=0 2024-12-03T09:25:16,550 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:16,550 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:16,551 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:16,554 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-20829414/hregion-20829414.1733217916531 2024-12-03T09:25:16,555 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:16,555 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 23258f7d89d4c4fa869babc3d5daf8ba, NAME => 'testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:16,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741838_1014 (size=64) 2024-12-03T09:25:16,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741838_1014 (size=64) 2024-12-03T09:25:16,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741838_1014 (size=64) 2024-12-03T09:25:16,572 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:16,574 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,576 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23258f7d89d4c4fa869babc3d5daf8ba columnFamilyName a 2024-12-03T09:25:16,576 DEBUG [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,577 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] regionserver.HStore(327): Store=23258f7d89d4c4fa869babc3d5daf8ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:16,578 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,579 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,580 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,580 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,580 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,583 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,587 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:16,588 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23258f7d89d4c4fa869babc3d5daf8ba; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66498507, jitterRate=-0.009095028042793274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23258f7d89d4c4fa869babc3d5daf8ba: Writing region info on filesystem at 1733217916572Initializing all the Stores at 1733217916573 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217916573Cleaning up temporary data from old regions at 1733217916580 (+7 ms)Region opened successfully at 1733217916589 (+9 ms) 2024-12-03T09:25:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 23258f7d89d4c4fa869babc3d5daf8ba, disabling compactions & flushes 2024-12-03T09:25:16,590 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. 2024-12-03T09:25:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. 2024-12-03T09:25:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. after waiting 0 ms 2024-12-03T09:25:16,590 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. 2024-12-03T09:25:16,590 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. 2024-12-03T09:25:16,591 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 23258f7d89d4c4fa869babc3d5daf8ba: Waiting for close lock at 1733217916590Disabling compacts and flushes for region at 1733217916590Disabling writes for close at 1733217916590Writing region close event to WAL at 1733217916590Closed at 1733217916590 2024-12-03T09:25:16,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741837_1013 (size=95) 2024-12-03T09:25:16,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741837_1013 (size=95) 2024-12-03T09:25:16,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741837_1013 (size=95) 2024-12-03T09:25:16,602 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:16,602 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-20829414:(num 1733217916531) 2024-12-03T09:25:16,604 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T09:25:16,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741839_1015 (size=320) 2024-12-03T09:25:16,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741839_1015 (size=320) 2024-12-03T09:25:16,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741839_1015 (size=320) 2024-12-03T09:25:16,624 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T09:25:16,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741840_1016 (size=253) 2024-12-03T09:25:16,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741840_1016 (size=253) 2024-12-03T09:25:16,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741840_1016 (size=253) 2024-12-03T09:25:16,655 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1, size=320 (320bytes) 2024-12-03T09:25:16,655 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T09:25:16,656 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T09:25:16,656 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1 2024-12-03T09:25:16,661 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1 after 3ms 2024-12-03T09:25:16,668 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:16,669 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1 took 14ms 2024-12-03T09:25:16,679 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1 so closing down 2024-12-03T09:25:16,679 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:16,682 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T09:25:16,684 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T09:25:16,686 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741841_1017 (size=320) 2024-12-03T09:25:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741841_1017 (size=320) 2024-12-03T09:25:16,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741841_1017 (size=320) 2024-12-03T09:25:16,697 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:16,700 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002 2024-12-03T09:25:16,704 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-03T09:25:16,704 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1, journal: Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1, size=320 (320bytes) at 1733217916655Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1 so closing down at 1733217916679 (+24 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp at 1733217916684 (+5 ms)3 split writer threads finished at 1733217916686 (+2 ms)Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733217916697 (+11 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002 at 1733217916700 (+3 ms)Processed 2 edits across 1 Regions in 30 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733217916704 (+4 ms) 2024-12-03T09:25:16,718 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2, size=253 (253bytes) 2024-12-03T09:25:16,718 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2 2024-12-03T09:25:16,719 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2 after 1ms 2024-12-03T09:25:16,723 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:16,723 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2 took 5ms 2024-12-03T09:25:16,726 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2 so closing down 2024-12-03T09:25:16,726 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:16,728 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T09:25:16,730 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T09:25:16,730 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:16,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741842_1018 (size=253) 2024-12-03T09:25:16,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741842_1018 (size=253) 2024-12-03T09:25:16,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741842_1018 (size=253) 2024-12-03T09:25:16,740 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:16,746 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:16,749 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002-wal-2.temp, length=253 2024-12-03T09:25:16,751 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-03T09:25:16,751 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2, journal: Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2, size=253 (253bytes) at 1733217916718Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2 so closing down at 1733217916726 (+8 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002-wal-2.temp at 1733217916730 (+4 ms)3 split writer threads finished at 1733217916730Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733217916740 (+10 ms)Processed 1 edits across 1 Regions in 26 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733217916751 (+11 ms) 2024-12-03T09:25:16,751 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:16,753 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:16,768 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal.1733217916755, exclude list is [], retry=0 2024-12-03T09:25:16,772 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:16,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:16,773 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:16,777 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal.1733217916755 2024-12-03T09:25:16,778 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:16,778 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 23258f7d89d4c4fa869babc3d5daf8ba, NAME => 'testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:16,779 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:16,779 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,779 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,781 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,783 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 23258f7d89d4c4fa869babc3d5daf8ba columnFamilyName a 2024-12-03T09:25:16,783 DEBUG [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:16,784 INFO [StoreOpener-23258f7d89d4c4fa869babc3d5daf8ba-1 {}] regionserver.HStore(327): Store=23258f7d89d4c4fa869babc3d5daf8ba/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:16,784 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,786 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,788 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,790 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002 2024-12-03T09:25:16,793 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:16,800 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002 2024-12-03T09:25:16,803 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 23258f7d89d4c4fa869babc3d5daf8ba 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T09:25:16,848 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/.tmp/a/7d9e0cf8440a4bdba8bbbed8a0085366 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733217916603/Put/seqid=0 2024-12-03T09:25:16,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741844_1020 (size=5170) 2024-12-03T09:25:16,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741844_1020 (size=5170) 2024-12-03T09:25:16,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741844_1020 (size=5170) 2024-12-03T09:25:16,861 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/.tmp/a/7d9e0cf8440a4bdba8bbbed8a0085366 2024-12-03T09:25:16,933 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/.tmp/a/7d9e0cf8440a4bdba8bbbed8a0085366 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/a/7d9e0cf8440a4bdba8bbbed8a0085366 2024-12-03T09:25:16,941 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/a/7d9e0cf8440a4bdba8bbbed8a0085366, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T09:25:16,948 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23258f7d89d4c4fa869babc3d5daf8ba in 142ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T09:25:16,950 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/0000000000000000002 2024-12-03T09:25:16,951 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,951 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,954 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 23258f7d89d4c4fa869babc3d5daf8ba 2024-12-03T09:25:16,957 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/23258f7d89d4c4fa869babc3d5daf8ba/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T09:25:16,958 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 23258f7d89d4c4fa869babc3d5daf8ba; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62299200, jitterRate=-0.0716695785522461}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:16,959 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 23258f7d89d4c4fa869babc3d5daf8ba: Writing region info on filesystem at 1733217916779Initializing all the Stores at 1733217916780 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217916781 (+1 ms)Obtaining lock to block concurrent updates at 1733217916803 (+22 ms)Preparing flush snapshotting stores in 23258f7d89d4c4fa869babc3d5daf8ba at 1733217916803Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733217916806 (+3 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733217916522.23258f7d89d4c4fa869babc3d5daf8ba. at 1733217916806Flushing 23258f7d89d4c4fa869babc3d5daf8ba/a: creating writer at 1733217916808 (+2 ms)Flushing 23258f7d89d4c4fa869babc3d5daf8ba/a: appending metadata at 1733217916840 (+32 ms)Flushing 23258f7d89d4c4fa869babc3d5daf8ba/a: closing flushed file at 1733217916842 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3e1596b5: reopening flushed file at 1733217916931 (+89 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 23258f7d89d4c4fa869babc3d5daf8ba in 142ms, sequenceid=2, compaction requested=false; wal=null at 1733217916948 (+17 ms)Cleaning up temporary data from old regions at 1733217916951 (+3 ms)Region opened successfully at 1733217916959 (+8 ms) 2024-12-03T09:25:16,979 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit0 Thread=380 (was 369) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:33280 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:33428 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:59006 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:36778 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:36664 [Waiting for operation #4] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:37427/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=13516 (was 13601) 2024-12-03T09:25:16,989 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=380, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=13515 2024-12-03T09:25:17,004 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:17,006 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:17,007 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:17,010 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-55413853, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-55413853, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:17,021 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-55413853/hregion-55413853.1733217917010, exclude list is [], retry=0 2024-12-03T09:25:17,024 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:17,024 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:17,025 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:17,027 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-55413853/hregion-55413853.1733217917010 2024-12-03T09:25:17,027 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:17,027 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 4955cd09e8ea5bbbdd0d485d9813c021, NAME => 'testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:17,038 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741846_1022 (size=64) 2024-12-03T09:25:17,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741846_1022 (size=64) 2024-12-03T09:25:17,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741846_1022 (size=64) 2024-12-03T09:25:17,040 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:17,041 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,044 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4955cd09e8ea5bbbdd0d485d9813c021 columnFamilyName a 2024-12-03T09:25:17,044 DEBUG [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:17,044 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] regionserver.HStore(327): Store=4955cd09e8ea5bbbdd0d485d9813c021/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:17,045 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,046 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,047 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,047 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,049 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,052 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:17,053 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4955cd09e8ea5bbbdd0d485d9813c021; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72902378, jitterRate=0.08633008599281311}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:17,053 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4955cd09e8ea5bbbdd0d485d9813c021: Writing region info on filesystem at 1733217917040Initializing all the Stores at 1733217917041 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217917041Cleaning up temporary data from old regions at 1733217917047 (+6 ms)Region opened successfully at 1733217917053 (+6 ms) 2024-12-03T09:25:17,054 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 4955cd09e8ea5bbbdd0d485d9813c021, disabling compactions & flushes 2024-12-03T09:25:17,054 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. 2024-12-03T09:25:17,054 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. 2024-12-03T09:25:17,054 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. after waiting 0 ms 2024-12-03T09:25:17,054 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. 2024-12-03T09:25:17,054 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. 2024-12-03T09:25:17,054 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 4955cd09e8ea5bbbdd0d485d9813c021: Waiting for close lock at 1733217917054Disabling compacts and flushes for region at 1733217917054Disabling writes for close at 1733217917054Writing region close event to WAL at 1733217917054Closed at 1733217917054 2024-12-03T09:25:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741845_1021 (size=95) 2024-12-03T09:25:17,058 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-55413853/hregion-55413853.1733217917010 not finished, retry = 0 2024-12-03T09:25:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741845_1021 (size=95) 2024-12-03T09:25:17,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741845_1021 (size=95) 2024-12-03T09:25:17,164 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:17,165 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-55413853:(num 1733217917010) 2024-12-03T09:25:17,167 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T09:25:17,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741847_1023 (size=320) 2024-12-03T09:25:17,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741847_1023 (size=320) 2024-12-03T09:25:17,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741847_1023 (size=320) 2024-12-03T09:25:17,185 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T09:25:17,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741848_1024 (size=253) 2024-12-03T09:25:17,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741848_1024 (size=253) 2024-12-03T09:25:17,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741848_1024 (size=253) 2024-12-03T09:25:17,212 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2, size=253 (253bytes) 2024-12-03T09:25:17,212 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2 2024-12-03T09:25:17,213 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2 after 1ms 2024-12-03T09:25:17,216 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:17,216 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2 took 5ms 2024-12-03T09:25:17,218 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2 so closing down 2024-12-03T09:25:17,218 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:17,220 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T09:25:17,222 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T09:25:17,222 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741849_1025 (size=253) 2024-12-03T09:25:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741849_1025 (size=253) 2024-12-03T09:25:17,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741849_1025 (size=253) 2024-12-03T09:25:17,230 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:17,231 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 2024-12-03T09:25:17,231 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2, size=253, length=253, corrupted=false, cancelled=false 2024-12-03T09:25:17,232 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2, journal: Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2, size=253 (253bytes) at 1733217917212Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2 so closing down at 1733217917218 (+6 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp at 1733217917222 (+4 ms)3 split writer threads finished at 1733217917222Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733217917230 (+8 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 at 1733217917231 (+1 ms)Processed 1 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-2, size=253, length=253, corrupted=false, cancelled=false at 1733217917232 (+1 ms) 2024-12-03T09:25:17,246 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1, size=320 (320bytes) 2024-12-03T09:25:17,246 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1 2024-12-03T09:25:17,247 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1 after 1ms 2024-12-03T09:25:17,250 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:17,250 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1 took 4ms 2024-12-03T09:25:17,253 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1 so closing down 2024-12-03T09:25:17,253 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:17,255 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T09:25:17,256 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T09:25:17,257 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:17,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741850_1026 (size=320) 2024-12-03T09:25:17,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741850_1026 (size=320) 2024-12-03T09:25:17,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741850_1026 (size=320) 2024-12-03T09:25:17,264 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:17,268 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:17,270 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002, length=253 2024-12-03T09:25:17,273 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 2024-12-03T09:25:17,273 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1, size=320, length=320, corrupted=false, cancelled=false 2024-12-03T09:25:17,273 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1, journal: Splitting hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1, size=320 (320bytes) at 1733217917246Finishing writing output for hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1 so closing down at 1733217917253 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp at 1733217917256 (+3 ms)3 split writer threads finished at 1733217917257 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733217917264 (+7 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 at 1733217917273 (+9 ms)Processed 2 edits across 1 Regions in 22 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal-1, size=320, length=320, corrupted=false, cancelled=false at 1733217917273 2024-12-03T09:25:17,273 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:17,275 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:17,286 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal.1733217917276, exclude list is [], retry=0 2024-12-03T09:25:17,290 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:17,290 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:17,291 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:17,293 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal.1733217917276 2024-12-03T09:25:17,293 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:17,293 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 4955cd09e8ea5bbbdd0d485d9813c021, NAME => 'testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,294 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,296 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,298 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4955cd09e8ea5bbbdd0d485d9813c021 columnFamilyName a 2024-12-03T09:25:17,298 DEBUG [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:17,299 INFO [StoreOpener-4955cd09e8ea5bbbdd0d485d9813c021-1 {}] regionserver.HStore(327): Store=4955cd09e8ea5bbbdd0d485d9813c021/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:17,299 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,300 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,302 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,304 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 2024-12-03T09:25:17,307 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:17,309 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 2024-12-03T09:25:17,309 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 4955cd09e8ea5bbbdd0d485d9813c021 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T09:25:17,325 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/.tmp/a/ec07a792885b4476b49eb6619e8696a8 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733217917165/Put/seqid=0 2024-12-03T09:25:17,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741852_1028 (size=5170) 2024-12-03T09:25:17,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741852_1028 (size=5170) 2024-12-03T09:25:17,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741852_1028 (size=5170) 2024-12-03T09:25:17,334 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/.tmp/a/ec07a792885b4476b49eb6619e8696a8 2024-12-03T09:25:17,342 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/.tmp/a/ec07a792885b4476b49eb6619e8696a8 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/a/ec07a792885b4476b49eb6619e8696a8 2024-12-03T09:25:17,350 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/a/ec07a792885b4476b49eb6619e8696a8, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T09:25:17,351 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4955cd09e8ea5bbbdd0d485d9813c021 in 41ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T09:25:17,351 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/0000000000000000002 2024-12-03T09:25:17,352 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,352 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,355 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 4955cd09e8ea5bbbdd0d485d9813c021 2024-12-03T09:25:17,358 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/4955cd09e8ea5bbbdd0d485d9813c021/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T09:25:17,359 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 4955cd09e8ea5bbbdd0d485d9813c021; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59949371, jitterRate=-0.10668475925922394}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:17,360 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 4955cd09e8ea5bbbdd0d485d9813c021: Writing region info on filesystem at 1733217917294Initializing all the Stores at 1733217917295 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217917295Obtaining lock to block concurrent updates at 1733217917309 (+14 ms)Preparing flush snapshotting stores in 4955cd09e8ea5bbbdd0d485d9813c021 at 1733217917309Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733217917309Flushing stores of testReplayEditsWrittenIntoWAL,,1733217917005.4955cd09e8ea5bbbdd0d485d9813c021. at 1733217917309Flushing 4955cd09e8ea5bbbdd0d485d9813c021/a: creating writer at 1733217917309Flushing 4955cd09e8ea5bbbdd0d485d9813c021/a: appending metadata at 1733217917323 (+14 ms)Flushing 4955cd09e8ea5bbbdd0d485d9813c021/a: closing flushed file at 1733217917324 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22aa7205: reopening flushed file at 1733217917341 (+17 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 4955cd09e8ea5bbbdd0d485d9813c021 in 41ms, sequenceid=2, compaction requested=false; wal=null at 1733217917351 (+10 ms)Cleaning up temporary data from old regions at 1733217917352 (+1 ms)Region opened successfully at 1733217917359 (+7 ms) 2024-12-03T09:25:17,379 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testNameConflictWhenSplit1 Thread=390 (was 380) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:33502 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:33280 [Waiting for operation #26] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:59072 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:36664 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:36846 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:58848 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=235 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=13498 (was 13515) 2024-12-03T09:25:17,390 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=390, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=235, ProcessCount=11, AvailableMemoryMB=13497 2024-12-03T09:25:17,408 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:17,411 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:17,411 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:17,415 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-95897354, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-95897354, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:17,427 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-95897354/hregion-95897354.1733217917415, exclude list is [], retry=0 2024-12-03T09:25:17,431 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:17,431 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:17,431 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:17,433 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-95897354/hregion-95897354.1733217917415 2024-12-03T09:25:17,434 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:17,434 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 56e44407a85544dac9bf5ef89be42929, NAME => 'testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:17,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741854_1030 (size=64) 2024-12-03T09:25:17,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741854_1030 (size=64) 2024-12-03T09:25:17,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741854_1030 (size=64) 2024-12-03T09:25:17,447 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:17,448 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,450 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName a 2024-12-03T09:25:17,450 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:17,450 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:17,450 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,452 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName b 2024-12-03T09:25:17,452 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:17,452 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:17,452 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,454 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName c 2024-12-03T09:25:17,454 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:17,455 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:17,455 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,456 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,457 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,457 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,458 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:17,459 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:17,462 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:17,462 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56e44407a85544dac9bf5ef89be42929; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64596931, jitterRate=-0.037430718541145325}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56e44407a85544dac9bf5ef89be42929: Writing region info on filesystem at 1733217917447Initializing all the Stores at 1733217917448 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217917448Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217917448Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217917448Cleaning up temporary data from old regions at 1733217917457 (+9 ms)Region opened successfully at 1733217917462 (+5 ms) 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56e44407a85544dac9bf5ef89be42929, disabling compactions & flushes 2024-12-03T09:25:17,463 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. after waiting 0 ms 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:17,463 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:17,463 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56e44407a85544dac9bf5ef89be42929: Waiting for close lock at 1733217917463Disabling compacts and flushes for region at 1733217917463Disabling writes for close at 1733217917463Writing region close event to WAL at 1733217917463Closed at 1733217917463 2024-12-03T09:25:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741853_1029 (size=95) 2024-12-03T09:25:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741853_1029 (size=95) 2024-12-03T09:25:17,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741853_1029 (size=95) 2024-12-03T09:25:17,470 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:17,470 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-95897354:(num 1733217917415) 2024-12-03T09:25:17,471 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:17,473 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:17,487 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, exclude list is [], retry=0 2024-12-03T09:25:17,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:17,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:17,490 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:17,492 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 2024-12-03T09:25:17,493 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:17,685 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, size=0 (0bytes) 2024-12-03T09:25:17,685 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 might be still open, length is 0 2024-12-03T09:25:17,685 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 2024-12-03T09:25:17,687 WARN [IPC Server handler 0 on default port 37427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T09:25:17,687 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 after 2ms 2024-12-03T09:25:18,465 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:59116 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:34311:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59116 dst: /127.0.0.1:34311 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:34311 remote=/127.0.0.1:59116]. Total timeout mills is 60000, 59191 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:18,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:33552 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:36933:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33552 dst: /127.0.0.1:36933 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:18,466 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:36876 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:41153:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:36876 dst: /127.0.0.1:41153 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:18,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741855_1032 (size=263633) 2024-12-03T09:25:18,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741855_1032 (size=263633) 2024-12-03T09:25:18,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741855_1032 (size=263633) 2024-12-03T09:25:21,683 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:25:21,688 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 after 4003ms 2024-12-03T09:25:21,694 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:21,695 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 took 4010ms 2024-12-03T09:25:21,700 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733217917474.temp 2024-12-03T09:25:21,702 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp 2024-12-03T09:25:21,738 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T09:25:21,834 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474; continuing. 2024-12-03T09:25:21,834 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 so closing down 2024-12-03T09:25:21,834 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:21,834 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741856_1033 (size=263641) 2024-12-03T09:25:21,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741856_1033 (size=263641) 2024-12-03T09:25:21,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741856_1033 (size=263641) 2024-12-03T09:25:21,838 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp (wrote 3002 edits, skipped 0 edits in 66 ms) 2024-12-03T09:25:21,840 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 2024-12-03T09:25:21,841 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T09:25:21,841 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, journal: Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, size=0 (0bytes) at 1733217917685Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp at 1733217921703 (+4018 ms)Split 1024 edits, skipped 0 edits. at 1733217921780 (+77 ms)Split 2048 edits, skipped 0 edits. at 1733217921811 (+31 ms)Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 so closing down at 1733217921834 (+23 ms)3 split writer threads finished at 1733217921834Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp (wrote 3002 edits, skipped 0 edits in 66 ms) at 1733217921839 (+5 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000000001-wal.1733217917474.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 at 1733217921840 (+1 ms)Processed 3002 edits across 1 Regions in 145 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474, size=0, length=0, corrupted=false, cancelled=false at 1733217921841 (+1 ms) 2024-12-03T09:25:21,843 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217917474 2024-12-03T09:25:21,844 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 2024-12-03T09:25:21,844 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:21,847 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:21,866 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217921848, exclude list is [], retry=0 2024-12-03T09:25:21,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:21,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:21,870 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:21,872 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217921848 2024-12-03T09:25:21,872 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:21,873 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:21,874 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,876 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName a 2024-12-03T09:25:21,876 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:21,877 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:21,878 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,879 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName b 2024-12-03T09:25:21,879 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:21,880 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:21,880 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,882 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 56e44407a85544dac9bf5ef89be42929 columnFamilyName c 2024-12-03T09:25:21,882 DEBUG [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:21,882 INFO [StoreOpener-56e44407a85544dac9bf5ef89be42929-1 {}] regionserver.HStore(327): Store=56e44407a85544dac9bf5ef89be42929/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:21,883 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,884 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,886 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:21,886 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 2024-12-03T09:25:21,892 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:21,944 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T09:25:22,273 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56e44407a85544dac9bf5ef89be42929 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T09:25:22,311 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/5ea057d1225046a5a2a9a6e8ef8aead4 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733217917499/Put/seqid=0 2024-12-03T09:25:22,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741858_1035 (size=50463) 2024-12-03T09:25:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741858_1035 (size=50463) 2024-12-03T09:25:22,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741858_1035 (size=50463) 2024-12-03T09:25:22,324 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/5ea057d1225046a5a2a9a6e8ef8aead4 2024-12-03T09:25:22,332 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/5ea057d1225046a5a2a9a6e8ef8aead4 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/a/5ea057d1225046a5a2a9a6e8ef8aead4 2024-12-03T09:25:22,339 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/a/5ea057d1225046a5a2a9a6e8ef8aead4, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T09:25:22,340 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 56e44407a85544dac9bf5ef89be42929 in 66ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T09:25:22,364 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T09:25:22,364 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56e44407a85544dac9bf5ef89be42929 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T09:25:22,373 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/beeb875591d94e8bb07764903cd7f692 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733217917534/Put/seqid=0 2024-12-03T09:25:22,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741859_1036 (size=20072) 2024-12-03T09:25:22,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741859_1036 (size=20072) 2024-12-03T09:25:22,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/beeb875591d94e8bb07764903cd7f692 2024-12-03T09:25:22,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741859_1036 (size=20072) 2024-12-03T09:25:22,410 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/4985418c7fdc4d56a80d077180d4ba6b is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733217917568/Put/seqid=0 2024-12-03T09:25:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741860_1037 (size=35835) 2024-12-03T09:25:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741860_1037 (size=35835) 2024-12-03T09:25:22,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741860_1037 (size=35835) 2024-12-03T09:25:22,421 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/4985418c7fdc4d56a80d077180d4ba6b 2024-12-03T09:25:22,430 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/a/beeb875591d94e8bb07764903cd7f692 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/a/beeb875591d94e8bb07764903cd7f692 2024-12-03T09:25:22,438 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/a/beeb875591d94e8bb07764903cd7f692, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T09:25:22,440 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/4985418c7fdc4d56a80d077180d4ba6b as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/b/4985418c7fdc4d56a80d077180d4ba6b 2024-12-03T09:25:22,449 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/b/4985418c7fdc4d56a80d077180d4ba6b, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T09:25:22,449 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 56e44407a85544dac9bf5ef89be42929 in 85ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T09:25:22,469 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T09:25:22,469 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56e44407a85544dac9bf5ef89be42929 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T09:25:22,476 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/87b7aa201c56418fa998e98349bf774a is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733217917583/Put/seqid=0 2024-12-03T09:25:22,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741861_1038 (size=35082) 2024-12-03T09:25:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741861_1038 (size=35082) 2024-12-03T09:25:22,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741861_1038 (size=35082) 2024-12-03T09:25:22,486 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/87b7aa201c56418fa998e98349bf774a 2024-12-03T09:25:22,506 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/3e9205685afc48ef816f0818a8e749b1 is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733217917612/Put/seqid=0 2024-12-03T09:25:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741862_1039 (size=20825) 2024-12-03T09:25:22,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741862_1039 (size=20825) 2024-12-03T09:25:22,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741862_1039 (size=20825) 2024-12-03T09:25:22,514 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/3e9205685afc48ef816f0818a8e749b1 2024-12-03T09:25:22,522 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/b/87b7aa201c56418fa998e98349bf774a as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/b/87b7aa201c56418fa998e98349bf774a 2024-12-03T09:25:22,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/b/87b7aa201c56418fa998e98349bf774a, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T09:25:22,531 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/3e9205685afc48ef816f0818a8e749b1 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/c/3e9205685afc48ef816f0818a8e749b1 2024-12-03T09:25:22,537 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/c/3e9205685afc48ef816f0818a8e749b1, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T09:25:22,538 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 56e44407a85544dac9bf5ef89be42929 in 69ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T09:25:22,548 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733217917654/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:22,550 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 2024-12-03T09:25:22,551 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T09:25:22,551 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 56e44407a85544dac9bf5ef89be42929 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T09:25:22,557 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/9b585ca8a31f42dca2e15d1aa96f2c35 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733217917619/Put/seqid=0 2024-12-03T09:25:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741863_1040 (size=50301) 2024-12-03T09:25:22,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741863_1040 (size=50301) 2024-12-03T09:25:22,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741863_1040 (size=50301) 2024-12-03T09:25:22,565 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/9b585ca8a31f42dca2e15d1aa96f2c35 2024-12-03T09:25:22,573 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b585ca8a31f42dca2e15d1aa96f2c35 2024-12-03T09:25:22,575 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/.tmp/c/9b585ca8a31f42dca2e15d1aa96f2c35 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/c/9b585ca8a31f42dca2e15d1aa96f2c35 2024-12-03T09:25:22,582 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 9b585ca8a31f42dca2e15d1aa96f2c35 2024-12-03T09:25:22,582 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/c/9b585ca8a31f42dca2e15d1aa96f2c35, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T09:25:22,583 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 56e44407a85544dac9bf5ef89be42929 in 31ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T09:25:22,584 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/0000000000000003002 2024-12-03T09:25:22,585 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:22,585 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:22,586 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T09:25:22,588 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 56e44407a85544dac9bf5ef89be42929 2024-12-03T09:25:22,591 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenIntoWAL/56e44407a85544dac9bf5ef89be42929/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T09:25:22,592 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 56e44407a85544dac9bf5ef89be42929; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69389384, jitterRate=0.03398239612579346}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T09:25:22,592 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 56e44407a85544dac9bf5ef89be42929: Writing region info on filesystem at 1733217921873Initializing all the Stores at 1733217921874 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217921874Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217921874Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217921874Cleaning up temporary data from old regions at 1733217922585 (+711 ms)Region opened successfully at 1733217922592 (+7 ms) 2024-12-03T09:25:22,656 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 56e44407a85544dac9bf5ef89be42929, disabling compactions & flushes 2024-12-03T09:25:22,656 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:22,656 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:22,656 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. after waiting 0 ms 2024-12-03T09:25:22,656 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:22,658 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733217917409.56e44407a85544dac9bf5ef89be42929. 2024-12-03T09:25:22,658 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 56e44407a85544dac9bf5ef89be42929: Waiting for close lock at 1733217922655Disabling compacts and flushes for region at 1733217922655Disabling writes for close at 1733217922656 (+1 ms)Writing region close event to WAL at 1733217922658 (+2 ms)Closed at 1733217922658 2024-12-03T09:25:22,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741857_1034 (size=95) 2024-12-03T09:25:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741857_1034 (size=95) 2024-12-03T09:25:22,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741857_1034 (size=95) 2024-12-03T09:25:22,665 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:22,666 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733217921848) 2024-12-03T09:25:22,680 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenIntoWAL Thread=404 (was 390) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:43269 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38187 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:37427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_205447679_22 at /127.0.0.1:40368 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:37427 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:43269 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:38187 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_205447679_22 at /127.0.0.1:40350 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_205447679_22 at /127.0.0.1:46322 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_205447679_22 at /127.0.0.1:36888 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 235), ProcessCount=11 (was 11), AvailableMemoryMB=13436 (was 13497) 2024-12-03T09:25:22,690 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=404, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=13436 2024-12-03T09:25:22,703 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:22,704 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:22,705 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:22,708 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-18032864, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-18032864, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:22,719 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-18032864/hregion-18032864.1733217922708, exclude list is [], retry=0 2024-12-03T09:25:22,722 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:22,722 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:22,722 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:22,726 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-18032864/hregion-18032864.1733217922708 2024-12-03T09:25:22,726 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:22,727 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a12ef621166a24482c7e8ee2ebc79e5f, NAME => 'test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:22,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741865_1042 (size=43) 2024-12-03T09:25:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741865_1042 (size=43) 2024-12-03T09:25:22,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741865_1042 (size=43) 2024-12-03T09:25:22,737 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:22,738 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,740 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName a 2024-12-03T09:25:22,740 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:22,741 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:22,741 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,743 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName b 2024-12-03T09:25:22,743 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:22,744 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:22,744 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,746 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName c 2024-12-03T09:25:22,746 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:22,747 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:22,747 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,748 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,748 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,750 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,750 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,750 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:22,752 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:22,754 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:22,755 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a12ef621166a24482c7e8ee2ebc79e5f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69082508, jitterRate=0.029409587383270264}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:22,756 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a12ef621166a24482c7e8ee2ebc79e5f: Writing region info on filesystem at 1733217922737Initializing all the Stores at 1733217922738 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217922738Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217922738Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217922738Cleaning up temporary data from old regions at 1733217922750 (+12 ms)Region opened successfully at 1733217922756 (+6 ms) 2024-12-03T09:25:22,757 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a12ef621166a24482c7e8ee2ebc79e5f, disabling compactions & flushes 2024-12-03T09:25:22,757 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:22,757 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:22,757 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. after waiting 0 ms 2024-12-03T09:25:22,757 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:22,757 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:22,757 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a12ef621166a24482c7e8ee2ebc79e5f: Waiting for close lock at 1733217922757Disabling compacts and flushes for region at 1733217922757Disabling writes for close at 1733217922757Writing region close event to WAL at 1733217922757Closed at 1733217922757 2024-12-03T09:25:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741864_1041 (size=95) 2024-12-03T09:25:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741864_1041 (size=95) 2024-12-03T09:25:22,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741864_1041 (size=95) 2024-12-03T09:25:22,764 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:22,764 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-18032864:(num 1733217922708) 2024-12-03T09:25:22,764 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:22,766 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:22,779 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, exclude list is [], retry=0 2024-12-03T09:25:22,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:22,782 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:22,783 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:22,785 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 2024-12-03T09:25:22,785 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741866_1043 (size=263359) 2024-12-03T09:25:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741866_1043 (size=263359) 2024-12-03T09:25:22,900 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741866_1043 (size=263359) 2024-12-03T09:25:22,916 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, size=257.2 K (263359bytes) 2024-12-03T09:25:22,916 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 2024-12-03T09:25:22,916 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 after 0ms 2024-12-03T09:25:22,919 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:22,921 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 took 5ms 2024-12-03T09:25:22,925 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733217922766.temp 2024-12-03T09:25:22,927 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp 2024-12-03T09:25:22,966 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 so closing down 2024-12-03T09:25:22,967 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:22,967 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741867_1044 (size=263359) 2024-12-03T09:25:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741867_1044 (size=263359) 2024-12-03T09:25:22,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741867_1044 (size=263359) 2024-12-03T09:25:22,970 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp (wrote 3000 edits, skipped 0 edits in 36 ms) 2024-12-03T09:25:22,972 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp to hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 2024-12-03T09:25:22,972 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, size=257.2 K, length=263359, corrupted=false, cancelled=false 2024-12-03T09:25:22,972 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, journal: Splitting hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, size=257.2 K (263359bytes) at 1733217922916Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp at 1733217922927 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733217922939 (+12 ms)Split 2048 edits, skipped 0 edits. at 1733217922954 (+15 ms)Finishing writing output for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 so closing down at 1733217922967 (+13 ms)3 split writer threads finished at 1733217922967Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp (wrote 3000 edits, skipped 0 edits in 36 ms) at 1733217922970 (+3 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000000001-wal.1733217922766.temp to hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 at 1733217922972 (+2 ms)Processed 3000 edits across 1 Regions in 51 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766, size=257.2 K, length=263359, corrupted=false, cancelled=false at 1733217922972 2024-12-03T09:25:22,974 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922766 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217922766 2024-12-03T09:25:22,975 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 2024-12-03T09:25:22,975 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:22,978 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:22,992 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, exclude list is [], retry=0 2024-12-03T09:25:22,995 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:22,996 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:22,996 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:22,998 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 2024-12-03T09:25:22,999 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741868_1045 (size=263486) 2024-12-03T09:25:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741868_1045 (size=263486) 2024-12-03T09:25:23,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741868_1045 (size=263486) 2024-12-03T09:25:23,173 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, size=257.3 K (263486bytes) 2024-12-03T09:25:23,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 2024-12-03T09:25:23,173 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 after 0ms 2024-12-03T09:25:23,176 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:23,178 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 took 5ms 2024-12-03T09:25:23,182 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733217922979.temp 2024-12-03T09:25:23,184 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp 2024-12-03T09:25:23,242 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 so closing down 2024-12-03T09:25:23,243 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:23,243 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:23,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741869_1046 (size=263486) 2024-12-03T09:25:23,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741869_1046 (size=263486) 2024-12-03T09:25:23,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741869_1046 (size=263486) 2024-12-03T09:25:23,247 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp (wrote 3000 edits, skipped 0 edits in 37 ms) 2024-12-03T09:25:23,248 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp to hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 2024-12-03T09:25:23,249 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, size=257.3 K, length=263486, corrupted=false, cancelled=false 2024-12-03T09:25:23,249 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, journal: Splitting hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, size=257.3 K (263486bytes) at 1733217923173Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp at 1733217923184 (+11 ms)Split 1024 edits, skipped 0 edits. at 1733217923201 (+17 ms)Split 2048 edits, skipped 0 edits. at 1733217923223 (+22 ms)Finishing writing output for hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 so closing down at 1733217923242 (+19 ms)3 split writer threads finished at 1733217923243 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp (wrote 3000 edits, skipped 0 edits in 37 ms) at 1733217923247 (+4 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003001-wal.1733217922979.temp to hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 at 1733217923248 (+1 ms)Processed 3000 edits across 1 Regions in 70 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979, size=257.3 K, length=263486, corrupted=false, cancelled=false at 1733217923249 (+1 ms) 2024-12-03T09:25:23,251 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217922979 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217922979 2024-12-03T09:25:23,252 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 2024-12-03T09:25:23,252 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:23,255 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/test2727-manual,16010,1733217922702, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:23,275 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217923256, exclude list is [], retry=0 2024-12-03T09:25:23,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:23,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:23,279 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:23,284 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733217922702/wal.1733217923256 2024-12-03T09:25:23,284 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:23,284 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a12ef621166a24482c7e8ee2ebc79e5f, NAME => 'test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:23,284 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:23,285 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,285 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,286 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,288 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName a 2024-12-03T09:25:23,288 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,289 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,289 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,290 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName b 2024-12-03T09:25:23,290 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,291 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,292 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,293 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a12ef621166a24482c7e8ee2ebc79e5f columnFamilyName c 2024-12-03T09:25:23,293 DEBUG [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,294 INFO [StoreOpener-a12ef621166a24482c7e8ee2ebc79e5f-1 {}] regionserver.HStore(327): Store=a12ef621166a24482c7e8ee2ebc79e5f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,294 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,297 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,298 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 2024-12-03T09:25:23,301 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:23,350 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 2024-12-03T09:25:23,351 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 2024-12-03T09:25:23,354 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:23,404 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 2024-12-03T09:25:23,404 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a12ef621166a24482c7e8ee2ebc79e5f 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T09:25:23,424 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/a/2253c291293d49dd8b03215fe45cb7f2 is 41, key is test2727/a:100/1733217923005/Put/seqid=0 2024-12-03T09:25:23,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741871_1048 (size=84227) 2024-12-03T09:25:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741871_1048 (size=84227) 2024-12-03T09:25:23,431 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741871_1048 (size=84227) 2024-12-03T09:25:23,432 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/a/2253c291293d49dd8b03215fe45cb7f2 2024-12-03T09:25:23,457 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/b/117fdc1bd68b4ff5b5a6f5819813e76e is 41, key is test2727/b:100/1733217923064/Put/seqid=0 2024-12-03T09:25:23,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741872_1049 (size=84609) 2024-12-03T09:25:23,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741872_1049 (size=84609) 2024-12-03T09:25:23,462 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741872_1049 (size=84609) 2024-12-03T09:25:23,463 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/b/117fdc1bd68b4ff5b5a6f5819813e76e 2024-12-03T09:25:23,496 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/c/7a1fc7f5937d458180eccec119ea8ce9 is 41, key is test2727/c:100/1733217923116/Put/seqid=0 2024-12-03T09:25:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741873_1050 (size=84609) 2024-12-03T09:25:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741873_1050 (size=84609) 2024-12-03T09:25:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741873_1050 (size=84609) 2024-12-03T09:25:23,503 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/c/7a1fc7f5937d458180eccec119ea8ce9 2024-12-03T09:25:23,511 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/a/2253c291293d49dd8b03215fe45cb7f2 as hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/a/2253c291293d49dd8b03215fe45cb7f2 2024-12-03T09:25:23,520 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/a/2253c291293d49dd8b03215fe45cb7f2, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T09:25:23,522 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/b/117fdc1bd68b4ff5b5a6f5819813e76e as hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/b/117fdc1bd68b4ff5b5a6f5819813e76e 2024-12-03T09:25:23,530 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/b/117fdc1bd68b4ff5b5a6f5819813e76e, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T09:25:23,532 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/.tmp/c/7a1fc7f5937d458180eccec119ea8ce9 as hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/c/7a1fc7f5937d458180eccec119ea8ce9 2024-12-03T09:25:23,541 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/c/7a1fc7f5937d458180eccec119ea8ce9, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T09:25:23,542 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a12ef621166a24482c7e8ee2ebc79e5f in 138ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T09:25:23,543 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000003000 2024-12-03T09:25:23,544 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/0000000000000006000 2024-12-03T09:25:23,545 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,546 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,546 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:23,548 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a12ef621166a24482c7e8ee2ebc79e5f 2024-12-03T09:25:23,552 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/test2727/a12ef621166a24482c7e8ee2ebc79e5f/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T09:25:23,553 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a12ef621166a24482c7e8ee2ebc79e5f; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69641756, jitterRate=0.03774303197860718}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:23,555 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a12ef621166a24482c7e8ee2ebc79e5f: Writing region info on filesystem at 1733217923285Initializing all the Stores at 1733217923286 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923286Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923286Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923286Obtaining lock to block concurrent updates at 1733217923404 (+118 ms)Preparing flush snapshotting stores in a12ef621166a24482c7e8ee2ebc79e5f at 1733217923404Finished memstore snapshotting test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733217923405 (+1 ms)Flushing stores of test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. at 1733217923405Flushing a12ef621166a24482c7e8ee2ebc79e5f/a: creating writer at 1733217923405Flushing a12ef621166a24482c7e8ee2ebc79e5f/a: appending metadata at 1733217923423 (+18 ms)Flushing a12ef621166a24482c7e8ee2ebc79e5f/a: closing flushed file at 1733217923423Flushing a12ef621166a24482c7e8ee2ebc79e5f/b: creating writer at 1733217923439 (+16 ms)Flushing a12ef621166a24482c7e8ee2ebc79e5f/b: appending metadata at 1733217923456 (+17 ms)Flushing a12ef621166a24482c7e8ee2ebc79e5f/b: closing flushed file at 1733217923456Flushing a12ef621166a24482c7e8ee2ebc79e5f/c: creating writer at 1733217923470 (+14 ms)Flushing a12ef621166a24482c7e8ee2ebc79e5f/c: appending metadata at 1733217923495 (+25 ms)Flushing a12ef621166a24482c7e8ee2ebc79e5f/c: closing flushed file at 1733217923495Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3064ebfd: reopening flushed file at 1733217923510 (+15 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@238b3364: reopening flushed file at 1733217923520 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7f39b184: reopening flushed file at 1733217923531 (+11 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for a12ef621166a24482c7e8ee2ebc79e5f in 138ms, sequenceid=6000, compaction requested=false; wal=null at 1733217923542 (+11 ms)Cleaning up temporary data from old regions at 1733217923546 (+4 ms)Region opened successfully at 1733217923555 (+9 ms) 2024-12-03T09:25:23,557 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T09:25:23,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a12ef621166a24482c7e8ee2ebc79e5f, disabling compactions & flushes 2024-12-03T09:25:23,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:23,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:23,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. after waiting 0 ms 2024-12-03T09:25:23,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:23,558 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733217922703.a12ef621166a24482c7e8ee2ebc79e5f. 2024-12-03T09:25:23,559 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a12ef621166a24482c7e8ee2ebc79e5f: Waiting for close lock at 1733217923557Disabling compacts and flushes for region at 1733217923557Disabling writes for close at 1733217923557Writing region close event to WAL at 1733217923558 (+1 ms)Closed at 1733217923558 2024-12-03T09:25:23,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741870_1047 (size=95) 2024-12-03T09:25:23,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741870_1047 (size=95) 2024-12-03T09:25:23,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741870_1047 (size=95) 2024-12-03T09:25:23,565 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:23,565 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733217923256) 2024-12-03T09:25:23,578 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#test2727 Thread=407 (was 404) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40368 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46394 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40350 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:39510 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=13289 (was 13436) 2024-12-03T09:25:23,590 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=407, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=13287 2024-12-03T09:25:23,605 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:23,612 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:23,613 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733217923613 2024-12-03T09:25:23,622 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 2024-12-03T09:25:23,623 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:23,625 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 8a4807870209935cfb7ea2389834b3f2, NAME => 'testSequentialEditLogSeqNum,,1733217923606.8a4807870209935cfb7ea2389834b3f2.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:23,625 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733217923606.8a4807870209935cfb7ea2389834b3f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:23,625 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,625 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,626 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2 doesn't exist for region: 8a4807870209935cfb7ea2389834b3f2 on table testSequentialEditLogSeqNum 2024-12-03T09:25:23,627 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 8a4807870209935cfb7ea2389834b3f2 on table testSequentialEditLogSeqNum 2024-12-03T09:25:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741875_1052 (size=62) 2024-12-03T09:25:23,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741875_1052 (size=62) 2024-12-03T09:25:23,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741875_1052 (size=62) 2024-12-03T09:25:23,639 INFO [StoreOpener-8a4807870209935cfb7ea2389834b3f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,641 INFO [StoreOpener-8a4807870209935cfb7ea2389834b3f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 8a4807870209935cfb7ea2389834b3f2 columnFamilyName a 2024-12-03T09:25:23,641 DEBUG [StoreOpener-8a4807870209935cfb7ea2389834b3f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,642 INFO [StoreOpener-8a4807870209935cfb7ea2389834b3f2-1 {}] regionserver.HStore(327): Store=8a4807870209935cfb7ea2389834b3f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,642 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,643 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,643 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,644 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,645 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 8a4807870209935cfb7ea2389834b3f2 2024-12-03T09:25:23,647 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:23,648 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 8a4807870209935cfb7ea2389834b3f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61069910, jitterRate=-0.08998742699623108}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:23,649 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 8a4807870209935cfb7ea2389834b3f2: Writing region info on filesystem at 1733217923625Initializing all the Stores at 1733217923639 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923639Cleaning up temporary data from old regions at 1733217923644 (+5 ms)Region opened successfully at 1733217923649 (+5 ms) 2024-12-03T09:25:23,661 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 8a4807870209935cfb7ea2389834b3f2 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T09:25:23,675 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T09:25:23,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:23,679 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T09:25:23,679 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T09:25:23,680 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T09:25:23,680 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:23,681 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T09:25:23,681 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:23,682 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T09:25:23,682 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:23,685 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/.tmp/a/909b4f8934d242008c755cd2da99bf45 is 81, key is testSequentialEditLogSeqNum/a:x0/1733217923649/Put/seqid=0 2024-12-03T09:25:23,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741876_1053 (size=5833) 2024-12-03T09:25:23,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741876_1053 (size=5833) 2024-12-03T09:25:23,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741876_1053 (size=5833) 2024-12-03T09:25:23,694 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/.tmp/a/909b4f8934d242008c755cd2da99bf45 2024-12-03T09:25:23,702 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/.tmp/a/909b4f8934d242008c755cd2da99bf45 as hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/a/909b4f8934d242008c755cd2da99bf45 2024-12-03T09:25:23,710 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/a/909b4f8934d242008c755cd2da99bf45, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T09:25:23,712 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 8a4807870209935cfb7ea2389834b3f2 in 50ms, sequenceid=13, compaction requested=false 2024-12-03T09:25:23,712 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 8a4807870209935cfb7ea2389834b3f2: 2024-12-03T09:25:23,718 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T09:25:23,718 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T09:25:23,718 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T09:25:23,718 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T09:25:23,718 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T09:25:23,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741874_1051 (size=1843) 2024-12-03T09:25:23,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741874_1051 (size=1843) 2024-12-03T09:25:23,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741874_1051 (size=1843) 2024-12-03T09:25:23,736 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613, size=1.8 K (1843bytes) 2024-12-03T09:25:23,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 2024-12-03T09:25:23,737 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 after 0ms 2024-12-03T09:25:23,740 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:23,741 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 took 5ms 2024-12-03T09:25:23,743 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 so closing down 2024-12-03T09:25:23,744 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:23,744 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733217923613.temp 2024-12-03T09:25:23,746 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp 2024-12-03T09:25:23,747 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741877_1054 (size=1477) 2024-12-03T09:25:23,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741877_1054 (size=1477) 2024-12-03T09:25:23,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741877_1054 (size=1477) 2024-12-03T09:25:23,754 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:23,756 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp to hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000020 2024-12-03T09:25:23,756 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613, size=1.8 K, length=1843, corrupted=false, cancelled=false 2024-12-03T09:25:23,756 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613, journal: Splitting hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613, size=1.8 K (1843bytes) at 1733217923737Finishing writing output for hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613 so closing down at 1733217923744 (+7 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp at 1733217923746 (+2 ms)3 split writer threads finished at 1733217923747 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733217923754 (+7 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000003-wal.1733217923613.temp to hdfs://localhost:37427/hbase/data/default/testSequentialEditLogSeqNum/8a4807870209935cfb7ea2389834b3f2/recovered.edits/0000000000000000020 at 1733217923756 (+2 ms)Processed 17 edits across 1 Regions in 15 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733217923605/wal.1733217923613, size=1.8 K, length=1843, corrupted=false, cancelled=false at 1733217923756 2024-12-03T09:25:23,772 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testSequentialEditLogSeqNum Thread=412 (was 407) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40368 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46394 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40350 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=216 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=13267 (was 13287) 2024-12-03T09:25:23,783 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=412, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=216, ProcessCount=11, AvailableMemoryMB=13266 2024-12-03T09:25:23,804 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:23,807 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:23,833 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:23,836 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85468113, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-85468113, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:23,849 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85468113/hregion-85468113.1733217923837, exclude list is [], retry=0 2024-12-03T09:25:23,853 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:23,853 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:23,853 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:23,856 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85468113/hregion-85468113.1733217923837 2024-12-03T09:25:23,856 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:23,857 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => dcc316c5ae9452c075bf066bf90424db, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741879_1056 (size=70) 2024-12-03T09:25:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741879_1056 (size=70) 2024-12-03T09:25:23,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741879_1056 (size=70) 2024-12-03T09:25:23,869 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:23,871 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,873 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName a 2024-12-03T09:25:23,873 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,874 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,874 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,876 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName b 2024-12-03T09:25:23,876 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,876 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,876 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,878 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName c 2024-12-03T09:25:23,878 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,879 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,879 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,880 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,880 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,882 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,882 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,883 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:23,884 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,888 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:23,888 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc316c5ae9452c075bf066bf90424db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71763440, jitterRate=0.06935858726501465}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc316c5ae9452c075bf066bf90424db: Writing region info on filesystem at 1733217923870Initializing all the Stores at 1733217923871 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923871Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923871Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923871Cleaning up temporary data from old regions at 1733217923882 (+11 ms)Region opened successfully at 1733217923890 (+8 ms) 2024-12-03T09:25:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dcc316c5ae9452c075bf066bf90424db, disabling compactions & flushes 2024-12-03T09:25:23,890 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:23,891 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. after waiting 0 ms 2024-12-03T09:25:23,891 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:23,891 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:23,891 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dcc316c5ae9452c075bf066bf90424db: Waiting for close lock at 1733217923890Disabling compacts and flushes for region at 1733217923890Disabling writes for close at 1733217923891 (+1 ms)Writing region close event to WAL at 1733217923891Closed at 1733217923891 2024-12-03T09:25:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741878_1055 (size=95) 2024-12-03T09:25:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741878_1055 (size=95) 2024-12-03T09:25:23,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741878_1055 (size=95) 2024-12-03T09:25:23,898 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:23,898 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85468113:(num 1733217923837) 2024-12-03T09:25:23,899 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:23,901 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:23,914 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, exclude list is [], retry=0 2024-12-03T09:25:23,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:23,917 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:23,918 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:23,919 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 2024-12-03T09:25:23,920 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dcc316c5ae9452c075bf066bf90424db, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,920 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,922 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,923 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName a 2024-12-03T09:25:23,923 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,924 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,924 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,925 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName b 2024-12-03T09:25:23,925 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,925 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,925 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,926 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName c 2024-12-03T09:25:23,926 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:23,927 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:23,927 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,927 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,928 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,929 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,929 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,930 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:23,931 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:23,932 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc316c5ae9452c075bf066bf90424db; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73455280, jitterRate=0.09456896781921387}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:23,933 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc316c5ae9452c075bf066bf90424db: Writing region info on filesystem at 1733217923920Initializing all the Stores at 1733217923921 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923922 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923922Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217923922Cleaning up temporary data from old regions at 1733217923930 (+8 ms)Region opened successfully at 1733217923933 (+3 ms) 2024-12-03T09:25:23,938 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733217923937/Put/seqid=0 2024-12-03T09:25:23,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741881_1058 (size=4826) 2024-12-03T09:25:23,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741881_1058 (size=4826) 2024-12-03T09:25:23,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741881_1058 (size=4826) 2024-12-03T09:25:23,946 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in dcc316c5ae9452c075bf066bf90424db/a 2024-12-03T09:25:23,956 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T09:25:23,956 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T09:25:23,956 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for dcc316c5ae9452c075bf066bf90424db: 2024-12-03T09:25:23,958 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/53d5dcc247114989813f515abf4a7ccd_SeqId_3_ 2024-12-03T09:25:23,959 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into dcc316c5ae9452c075bf066bf90424db/a as hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/53d5dcc247114989813f515abf4a7ccd_SeqId_3_ - updating store file list. 2024-12-03T09:25:23,965 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 53d5dcc247114989813f515abf4a7ccd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:23,965 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/53d5dcc247114989813f515abf4a7ccd_SeqId_3_ into dcc316c5ae9452c075bf066bf90424db/a 2024-12-03T09:25:23,965 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into dcc316c5ae9452c075bf066bf90424db/a (new location: hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/53d5dcc247114989813f515abf4a7ccd_SeqId_3_) 2024-12-03T09:25:24,004 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, size=0 (0bytes) 2024-12-03T09:25:24,004 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 might be still open, length is 0 2024-12-03T09:25:24,004 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 2024-12-03T09:25:24,005 WARN [IPC Server handler 4 on default port 37427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T09:25:24,005 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 after 1ms 2024-12-03T09:25:24,461 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46462 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:36933:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46462 dst: /127.0.0.1:36933 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:36933 remote=/127.0.0.1:46462]. Total timeout mills is 60000, 59510 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:24,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:39578 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:41153:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39578 dst: /127.0.0.1:41153 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:24,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40590 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:34311:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40590 dst: /127.0.0.1:34311 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741880_1059 (size=475) 2024-12-03T09:25:24,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741880_1059 (size=475) 2024-12-03T09:25:28,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 after 4002ms 2024-12-03T09:25:28,013 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:28,014 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 took 4010ms 2024-12-03T09:25:28,017 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901; continuing. 2024-12-03T09:25:28,017 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 so closing down 2024-12-03T09:25:28,017 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:28,019 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733217923901.temp 2024-12-03T09:25:28,021 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp 2024-12-03T09:25:28,021 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:28,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741882_1060 (size=259) 2024-12-03T09:25:28,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741882_1060 (size=259) 2024-12-03T09:25:28,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741882_1060 (size=259) 2024-12-03T09:25:28,028 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:28,030 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp to hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 2024-12-03T09:25:28,030 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T09:25:28,030 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, journal: Splitting hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, size=0 (0bytes) at 1733217924004Finishing writing output for hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 so closing down at 1733217928017 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp at 1733217928021 (+4 ms)3 split writer threads finished at 1733217928021Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733217928028 (+7 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005-wal.1733217923901.temp to hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 at 1733217928030 (+2 ms)Processed 2 edits across 1 Regions in 15 ms; skipped=1; WAL=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901, size=0, length=0, corrupted=false, cancelled=false at 1733217928030 2024-12-03T09:25:28,032 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217923901 2024-12-03T09:25:28,033 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 2024-12-03T09:25:28,034 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:28,036 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:28,051 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217928037, exclude list is [], retry=0 2024-12-03T09:25:28,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:28,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:28,054 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:28,076 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217928037 2024-12-03T09:25:28,077 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:28,077 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => dcc316c5ae9452c075bf066bf90424db, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:28,077 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:28,077 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,077 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,079 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,080 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName a 2024-12-03T09:25:28,080 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,086 DEBUG [StoreFileOpener-dcc316c5ae9452c075bf066bf90424db-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 53d5dcc247114989813f515abf4a7ccd_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:28,086 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/53d5dcc247114989813f515abf4a7ccd_SeqId_3_ 2024-12-03T09:25:28,086 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,086 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,087 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName b 2024-12-03T09:25:28,087 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,088 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,088 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,089 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region dcc316c5ae9452c075bf066bf90424db columnFamilyName c 2024-12-03T09:25:28,089 DEBUG [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,089 INFO [StoreOpener-dcc316c5ae9452c075bf066bf90424db-1 {}] regionserver.HStore(327): Store=dcc316c5ae9452c075bf066bf90424db/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,089 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,090 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,091 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,092 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 2024-12-03T09:25:28,094 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:28,094 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 2024-12-03T09:25:28,094 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing dcc316c5ae9452c075bf066bf90424db 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T09:25:28,109 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/.tmp/a/440bdf80489849598031185a4826115e is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733217923970/Put/seqid=0 2024-12-03T09:25:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741884_1062 (size=5149) 2024-12-03T09:25:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741884_1062 (size=5149) 2024-12-03T09:25:28,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741884_1062 (size=5149) 2024-12-03T09:25:28,116 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/.tmp/a/440bdf80489849598031185a4826115e 2024-12-03T09:25:28,122 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/.tmp/a/440bdf80489849598031185a4826115e as hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/440bdf80489849598031185a4826115e 2024-12-03T09:25:28,127 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/a/440bdf80489849598031185a4826115e, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T09:25:28,128 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for dcc316c5ae9452c075bf066bf90424db in 34ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T09:25:28,128 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/0000000000000000005 2024-12-03T09:25:28,130 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,130 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,130 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:28,132 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for dcc316c5ae9452c075bf066bf90424db 2024-12-03T09:25:28,134 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/dcc316c5ae9452c075bf066bf90424db/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T09:25:28,135 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened dcc316c5ae9452c075bf066bf90424db; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68637091, jitterRate=0.022772356867790222}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:28,135 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for dcc316c5ae9452c075bf066bf90424db: Writing region info on filesystem at 1733217928077Initializing all the Stores at 1733217928078 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217928078Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217928078Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217928079 (+1 ms)Obtaining lock to block concurrent updates at 1733217928095 (+16 ms)Preparing flush snapshotting stores in dcc316c5ae9452c075bf066bf90424db at 1733217928095Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733217928095Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. at 1733217928095Flushing dcc316c5ae9452c075bf066bf90424db/a: creating writer at 1733217928095Flushing dcc316c5ae9452c075bf066bf90424db/a: appending metadata at 1733217928108 (+13 ms)Flushing dcc316c5ae9452c075bf066bf90424db/a: closing flushed file at 1733217928108Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53d6f000: reopening flushed file at 1733217928121 (+13 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for dcc316c5ae9452c075bf066bf90424db in 34ms, sequenceid=5, compaction requested=false; wal=null at 1733217928128 (+7 ms)Cleaning up temporary data from old regions at 1733217928130 (+2 ms)Region opened successfully at 1733217928135 (+5 ms) 2024-12-03T09:25:28,140 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing dcc316c5ae9452c075bf066bf90424db, disabling compactions & flushes 2024-12-03T09:25:28,140 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:28,140 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:28,140 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. after waiting 0 ms 2024-12-03T09:25:28,140 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:28,141 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733217923805.dcc316c5ae9452c075bf066bf90424db. 2024-12-03T09:25:28,141 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for dcc316c5ae9452c075bf066bf90424db: Waiting for close lock at 1733217928140Disabling compacts and flushes for region at 1733217928140Disabling writes for close at 1733217928140Writing region close event to WAL at 1733217928141 (+1 ms)Closed at 1733217928141 2024-12-03T09:25:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741883_1061 (size=95) 2024-12-03T09:25:28,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741883_1061 (size=95) 2024-12-03T09:25:28,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741883_1061 (size=95) 2024-12-03T09:25:28,146 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:28,146 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733217928037) 2024-12-03T09:25:28,158 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testRegionMadeOfBulkLoadedFilesOnly Thread=418 (was 412) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:37427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1450077469_22 at /127.0.0.1:39626 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:46303 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:37427 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:46303 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1450077469_22 at /127.0.0.1:46484 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1027 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=199 (was 216), ProcessCount=11 (was 11), AvailableMemoryMB=13169 (was 13266) 2024-12-03T09:25:28,160 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1027 is superior to 1024 2024-12-03T09:25:28,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=418, OpenFileDescriptor=1027, MaxFileDescriptor=1048576, SystemLoadAverage=199, ProcessCount=11, AvailableMemoryMB=13168 2024-12-03T09:25:28,172 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1027 is superior to 1024 2024-12-03T09:25:28,186 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:28,190 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T09:25:28,193 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 699c9baf3d29,33521,1733217913369 2024-12-03T09:25:28,195 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@41fa2aba 2024-12-03T09:25:28,196 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T09:25:28,198 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45248, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T09:25:28,202 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T09:25:28,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T09:25:28,210 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T09:25:28,212 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T09:25:28,212 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,214 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T09:25:28,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:25:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741885_1063 (size=694) 2024-12-03T09:25:28,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741885_1063 (size=694) 2024-12-03T09:25:28,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741885_1063 (size=694) 2024-12-03T09:25:28,226 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41 2024-12-03T09:25:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741886_1064 (size=77) 2024-12-03T09:25:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741886_1064 (size=77) 2024-12-03T09:25:28,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741886_1064 (size=77) 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:28,235 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,235 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,235 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217928235Disabling compacts and flushes for region at 1733217928235Disabling writes for close at 1733217928235Writing region close event to WAL at 1733217928235Closed at 1733217928235 2024-12-03T09:25:28,237 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T09:25:28,242 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733217928237"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733217928237"}]},"ts":"1733217928237"} 2024-12-03T09:25:28,245 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T09:25:28,246 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T09:25:28,249 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217928246"}]},"ts":"1733217928246"} 2024-12-03T09:25:28,252 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T09:25:28,252 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {699c9baf3d29=0} racks are {/default-rack=0} 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T09:25:28,254 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T09:25:28,254 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T09:25:28,254 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T09:25:28,254 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T09:25:28,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN}] 2024-12-03T09:25:28,257 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN 2024-12-03T09:25:28,258 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN; state=OFFLINE, location=699c9baf3d29,40779,1733217914054; forceNewPlan=false, retain=false 2024-12-03T09:25:28,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:25:28,411 INFO [699c9baf3d29:33521 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T09:25:28,412 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPENING, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:28,417 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN because future has completed 2024-12-03T09:25:28,418 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054}] 2024-12-03T09:25:28,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:25:28,582 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,583 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:28,583 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,583 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:28,584 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,584 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,586 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,588 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:28,589 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,589 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,590 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,592 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:28,592 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,593 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,593 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,594 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,595 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,596 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,596 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,597 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T09:25:28,599 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,602 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:28,603 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 60a49919b1e567cff342967e23c2879b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63252400, jitterRate=-0.05746579170227051}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T09:25:28,603 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,604 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 60a49919b1e567cff342967e23c2879b: Running coprocessor pre-open hook at 1733217928584Writing region info on filesystem at 1733217928584Initializing all the Stores at 1733217928585 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217928585Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217928586 (+1 ms)Cleaning up temporary data from old regions at 1733217928596 (+10 ms)Running coprocessor post-open hooks at 1733217928603 (+7 ms)Region opened successfully at 1733217928603 2024-12-03T09:25:28,606 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., pid=6, masterSystemTime=1733217928573 2024-12-03T09:25:28,609 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,609 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,610 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPEN, openSeqNum=2, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:28,613 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 because future has completed 2024-12-03T09:25:28,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T09:25:28,618 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 in 197 msec 2024-12-03T09:25:28,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T09:25:28,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN in 363 msec 2024-12-03T09:25:28,622 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T09:25:28,622 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733217928622"}]},"ts":"1733217928622"} 2024-12-03T09:25:28,625 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T09:25:28,626 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T09:25:28,628 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 423 msec 2024-12-03T09:25:28,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T09:25:28,851 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T09:25:28,852 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T09:25:28,855 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T09:25:28,864 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T09:25:28,864 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T09:25:28,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T09:25:28,876 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=2] 2024-12-03T09:25:28,892 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=60a49919b1e567cff342967e23c2879b, source=699c9baf3d29,40779,1733217914054, destination=699c9baf3d29,40727,1733217914150, warming up region on 699c9baf3d29,40727,1733217914150 2024-12-03T09:25:28,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:25:28,895 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=60a49919b1e567cff342967e23c2879b, source=699c9baf3d29,40779,1733217914054, destination=699c9baf3d29,40727,1733217914150, running balancer 2024-12-03T09:25:28,896 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42251, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:25:28,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE 2024-12-03T09:25:28,896 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE 2024-12-03T09:25:28,899 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=CLOSING, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:28,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(7855): Warmup {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:28,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:28,901 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE because future has completed 2024-12-03T09:25:28,901 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,902 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:25:28,902 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054}] 2024-12-03T09:25:28,903 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:28,903 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,904 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,904 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:28,905 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:28,905 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:28,906 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:28,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:28,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217928906Disabling compacts and flushes for region at 1733217928906Disabling writes for close at 1733217928906Writing region close event to WAL at 1733217928907 (+1 ms)Closed at 1733217928907 2024-12-03T09:25:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T09:25:29,060 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,061 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:25:29,061 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:29,061 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,061 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,062 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:29,062 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,062 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 60a49919b1e567cff342967e23c2879b 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T09:25:29,084 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/a6f3e13cfa234a9a912be546b5bfc65a is 35, key is r1/cf1:q/1733217928878/Put/seqid=0 2024-12-03T09:25:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741887_1065 (size=4783) 2024-12-03T09:25:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741887_1065 (size=4783) 2024-12-03T09:25:29,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741887_1065 (size=4783) 2024-12-03T09:25:29,092 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/a6f3e13cfa234a9a912be546b5bfc65a 2024-12-03T09:25:29,100 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/a6f3e13cfa234a9a912be546b5bfc65a as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a 2024-12-03T09:25:29,107 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T09:25:29,108 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 60a49919b1e567cff342967e23c2879b in 46ms, sequenceid=5, compaction requested=false 2024-12-03T09:25:29,108 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T09:25:29,114 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T09:25:29,116 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,117 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217929061Running coprocessor pre-close hooks at 1733217929061Disabling compacts and flushes for region at 1733217929061Disabling writes for close at 1733217929062 (+1 ms)Obtaining lock to block concurrent updates at 1733217929062Preparing flush snapshotting stores in 60a49919b1e567cff342967e23c2879b at 1733217929062Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733217929062Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. at 1733217929063 (+1 ms)Flushing 60a49919b1e567cff342967e23c2879b/cf1: creating writer at 1733217929063Flushing 60a49919b1e567cff342967e23c2879b/cf1: appending metadata at 1733217929084 (+21 ms)Flushing 60a49919b1e567cff342967e23c2879b/cf1: closing flushed file at 1733217929084Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@16165041: reopening flushed file at 1733217929098 (+14 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 60a49919b1e567cff342967e23c2879b in 46ms, sequenceid=5, compaction requested=false at 1733217929108 (+10 ms)Writing region close event to WAL at 1733217929110 (+2 ms)Running coprocessor post-close hooks at 1733217929114 (+4 ms)Closed at 1733217929116 (+2 ms) 2024-12-03T09:25:29,117 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 60a49919b1e567cff342967e23c2879b move to 699c9baf3d29,40727,1733217914150 record at close sequenceid=5 2024-12-03T09:25:29,120 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,121 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=CLOSED 2024-12-03T09:25:29,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 because future has completed 2024-12-03T09:25:29,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T09:25:29,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 in 223 msec 2024-12-03T09:25:29,130 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE; state=CLOSED, location=699c9baf3d29,40727,1733217914150; forceNewPlan=false, retain=false 2024-12-03T09:25:29,280 INFO [699c9baf3d29:33521 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T09:25:29,281 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPENING, regionLocation=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:29,284 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE because future has completed 2024-12-03T09:25:29,284 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150}] 2024-12-03T09:25:29,443 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,444 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:29,444 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,444 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:29,444 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,444 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,446 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,447 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:29,447 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:29,454 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a 2024-12-03T09:25:29,454 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:29,454 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,455 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:29,455 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:29,456 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:29,456 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,457 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,458 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,459 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,459 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,460 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T09:25:29,462 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,463 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 60a49919b1e567cff342967e23c2879b; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70901161, jitterRate=0.056509628891944885}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T09:25:29,463 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,463 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 60a49919b1e567cff342967e23c2879b: Running coprocessor pre-open hook at 1733217929444Writing region info on filesystem at 1733217929444Initializing all the Stores at 1733217929446 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217929446Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217929446Cleaning up temporary data from old regions at 1733217929459 (+13 ms)Running coprocessor post-open hooks at 1733217929463 (+4 ms)Region opened successfully at 1733217929463 2024-12-03T09:25:29,465 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., pid=9, masterSystemTime=1733217929439 2024-12-03T09:25:29,467 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,467 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,468 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPEN, openSeqNum=9, regionLocation=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:29,473 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150 because future has completed 2024-12-03T09:25:29,474 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=33521 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=699c9baf3d29,40727,1733217914150, table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-12-03T09:25:29,478 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T09:25:29,479 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150 in 191 msec 2024-12-03T09:25:29,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE in 584 msec 2024-12-03T09:25:29,503 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:25:29,505 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51580, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:25:29,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.2:45858 deadline: 1733217989509, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40727 startCode=1733217914150. As of locationSeqNum=5. 2024-12-03T09:25:29,531 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40727 startCode=1733217914150. As of locationSeqNum=5. 2024-12-03T09:25:29,532 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40727 startCode=1733217914150. As of locationSeqNum=5. 2024-12-03T09:25:29,532 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40727,1733217914150, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40727 startCode=1733217914150. As of locationSeqNum=5. 2024-12-03T09:25:29,638 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:25:29,639 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51582, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:25:29,648 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 60a49919b1e567cff342967e23c2879b 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T09:25:29,668 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 is 29, key is r1/cf1:/1733217929640/DeleteFamily/seqid=0 2024-12-03T09:25:29,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741888_1066 (size=4906) 2024-12-03T09:25:29,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741888_1066 (size=4906) 2024-12-03T09:25:29,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741888_1066 (size=4906) 2024-12-03T09:25:29,675 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,681 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,694 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf2/11c1674e78824cc48e14571d195c2205 is 29, key is r1/cf2:/1733217929640/DeleteFamily/seqid=0 2024-12-03T09:25:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741889_1067 (size=4906) 2024-12-03T09:25:29,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741889_1067 (size=4906) 2024-12-03T09:25:29,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741889_1067 (size=4906) 2024-12-03T09:25:29,701 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf2/11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,706 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,708 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,714 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,714 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T09:25:29,715 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf2/11c1674e78824cc48e14571d195c2205 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,722 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,722 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T09:25:29,723 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 60a49919b1e567cff342967e23c2879b in 75ms, sequenceid=12, compaction requested=false 2024-12-03T09:25:29,723 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 60a49919b1e567cff342967e23c2879b: 2024-12-03T09:25:29,726 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T09:25:29,727 DEBUG [Time-limited test {}] regionserver.HStore(1541): 60a49919b1e567cff342967e23c2879b/cf1 is initiating major compaction (all files) 2024-12-03T09:25:29,727 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:25:29,727 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:29,728 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 60a49919b1e567cff342967e23c2879b/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,728 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a, hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927] into tmpdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp, totalSize=9.5 K 2024-12-03T09:25:29,729 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a6f3e13cfa234a9a912be546b5bfc65a, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733217928878 2024-12-03T09:25:29,730 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 70bf8e7cc6c74127b20d2bdc4e7bf927, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T09:25:29,741 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 60a49919b1e567cff342967e23c2879b#cf1#compaction#16 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:25:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741890_1068 (size=4626) 2024-12-03T09:25:29,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741890_1068 (size=4626) 2024-12-03T09:25:29,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741890_1068 (size=4626) 2024-12-03T09:25:29,756 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf1/a1a14405012a47e099b2a7a58bcd2a37 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a1a14405012a47e099b2a7a58bcd2a37 2024-12-03T09:25:29,769 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 60a49919b1e567cff342967e23c2879b/cf1 of 60a49919b1e567cff342967e23c2879b into a1a14405012a47e099b2a7a58bcd2a37(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:25:29,769 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 60a49919b1e567cff342967e23c2879b: 2024-12-03T09:25:29,769 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T09:25:29,769 DEBUG [Time-limited test {}] regionserver.HStore(1541): 60a49919b1e567cff342967e23c2879b/cf2 is initiating major compaction (all files) 2024-12-03T09:25:29,769 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T09:25:29,769 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T09:25:29,770 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 60a49919b1e567cff342967e23c2879b/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,770 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205] into tmpdir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp, totalSize=4.8 K 2024-12-03T09:25:29,770 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 11c1674e78824cc48e14571d195c2205, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T09:25:29,776 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 60a49919b1e567cff342967e23c2879b#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T09:25:29,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741891_1069 (size=4592) 2024-12-03T09:25:29,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741891_1069 (size=4592) 2024-12-03T09:25:29,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741891_1069 (size=4592) 2024-12-03T09:25:29,790 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/.tmp/cf2/4fe56521b1d44b1faa6e2cd8688b3a84 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/4fe56521b1d44b1faa6e2cd8688b3a84 2024-12-03T09:25:29,797 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 60a49919b1e567cff342967e23c2879b/cf2 of 60a49919b1e567cff342967e23c2879b into 4fe56521b1d44b1faa6e2cd8688b3a84(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:25:29,797 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 60a49919b1e567cff342967e23c2879b: 2024-12-03T09:25:29,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(2410): Client=jenkins//172.17.0.2 move hri=60a49919b1e567cff342967e23c2879b, source=699c9baf3d29,40727,1733217914150, destination=699c9baf3d29,40779,1733217914054, warming up region on 699c9baf3d29,40779,1733217914054 2024-12-03T09:25:29,801 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(2414): Client=jenkins//172.17.0.2 move hri=60a49919b1e567cff342967e23c2879b, source=699c9baf3d29,40727,1733217914150, destination=699c9baf3d29,40779,1733217914054, running balancer 2024-12-03T09:25:29,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE 2024-12-03T09:25:29,803 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE 2024-12-03T09:25:29,804 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=CLOSING, regionLocation=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:29,804 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(7855): Warmup {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:29,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:29,805 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,807 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:29,807 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:29,807 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE because future has completed 2024-12-03T09:25:29,808 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T09:25:29,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150}] 2024-12-03T09:25:29,815 INFO [StoreFileOpener-60a49919b1e567cff342967e23c2879b-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,816 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,821 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a1a14405012a47e099b2a7a58bcd2a37 2024-12-03T09:25:29,828 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a 2024-12-03T09:25:29,828 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:29,828 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,830 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:29,830 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:29,838 INFO [StoreFileOpener-60a49919b1e567cff342967e23c2879b-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,838 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,845 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/4fe56521b1d44b1faa6e2cd8688b3a84 2024-12-03T09:25:29,845 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:29,845 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:29,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,846 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40779 {}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217929845Disabling compacts and flushes for region at 1733217929845Disabling writes for close at 1733217929845Writing region close event to WAL at 1733217929846 (+1 ms)Closed at 1733217929846 2024-12-03T09:25:29,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T09:25:29,963 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,963 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T09:25:29,963 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:29,963 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,963 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,963 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:29,963 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,964 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a, hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927] to archive 2024-12-03T09:25:29,967 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:25:29,970 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a6f3e13cfa234a9a912be546b5bfc65a 2024-12-03T09:25:29,972 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/70bf8e7cc6c74127b20d2bdc4e7bf927 2024-12-03T09:25:29,984 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205] to archive 2024-12-03T09:25:29,985 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:25:29,988 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205 to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/11c1674e78824cc48e14571d195c2205 2024-12-03T09:25:29,993 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T09:25:29,994 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:29,994 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217929963Running coprocessor pre-close hooks at 1733217929963Disabling compacts and flushes for region at 1733217929963Disabling writes for close at 1733217929963Writing region close event to WAL at 1733217929989 (+26 ms)Running coprocessor post-close hooks at 1733217929994 (+5 ms)Closed at 1733217929994 2024-12-03T09:25:29,994 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 60a49919b1e567cff342967e23c2879b move to 699c9baf3d29,40779,1733217914054 record at close sequenceid=12 2024-12-03T09:25:29,997 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:29,998 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=CLOSED 2024-12-03T09:25:30,001 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150 because future has completed 2024-12-03T09:25:30,006 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T09:25:30,007 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40727,1733217914150 in 195 msec 2024-12-03T09:25:30,008 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE; state=CLOSED, location=699c9baf3d29,40779,1733217914054; forceNewPlan=false, retain=false 2024-12-03T09:25:30,159 INFO [699c9baf3d29:33521 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T09:25:30,159 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPENING, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:30,165 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE because future has completed 2024-12-03T09:25:30,165 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054}] 2024-12-03T09:25:30,323 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,324 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:30,324 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,324 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:30,325 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,325 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,327 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,329 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:30,329 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:30,339 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a1a14405012a47e099b2a7a58bcd2a37 2024-12-03T09:25:30,339 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:30,339 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,340 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:30,340 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:30,347 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/4fe56521b1d44b1faa6e2cd8688b3a84 2024-12-03T09:25:30,347 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:30,348 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,349 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,350 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,351 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T09:25:30,353 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,353 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 60a49919b1e567cff342967e23c2879b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73889953, jitterRate=0.10104610025882721}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T09:25:30,354 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,354 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 60a49919b1e567cff342967e23c2879b: Running coprocessor pre-open hook at 1733217930325Writing region info on filesystem at 1733217930325Initializing all the Stores at 1733217930326 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217930326Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217930327 (+1 ms)Cleaning up temporary data from old regions at 1733217930351 (+24 ms)Running coprocessor post-open hooks at 1733217930354 (+3 ms)Region opened successfully at 1733217930354 2024-12-03T09:25:30,355 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., pid=12, masterSystemTime=1733217930318 2024-12-03T09:25:30,357 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,357 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,358 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPEN, openSeqNum=18, regionLocation=699c9baf3d29,40779,1733217914054 2024-12-03T09:25:30,360 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 because future has completed 2024-12-03T09:25:30,363 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T09:25:30,364 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,40779,1733217914054 in 196 msec 2024-12-03T09:25:30,365 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, REOPEN/MOVE in 563 msec 2024-12-03T09:25:30,405 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:25:30,406 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:55612, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:25:30,408 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 699c9baf3d29,40779,1733217914054: testing ***** 2024-12-03T09:25:30,408 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T09:25:30,410 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T09:25:30,410 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T09:25:30,413 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T09:25:30,414 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T09:25:30,419 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 272839168 }, "NonHeapMemoryUsage": { "committed": 170917888, "init": 7667712, "max": -1, "used": 168233384 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "699c9baf3d29", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 3, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 1, "ProcessCallTime_max": 10, "ProcessCallTime_mean": 3, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 5, "ProcessCallTime_75th_percentile": 7, "ProcessCallTime_90th_percentile": 9, "ProcessCallTime_95th_percentile": 9, "ProcessCallTime_98th_percentile": 9, "ProcessCallTime_99th_percentile": 9, "ProcessCallTime_99.9th_percentile": 9, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 1, "TotalCallTime_max": 10, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 5, "TotalCallTime_75th_percentile": 7, "TotalCallTime_90th_percentile": 9, "TotalCallTime_95th_percentile": 9, "TotalCallTime_98th_percentile": 9, "TotalCallTime_99th_percentile": 9, "TotalCallTime_99.9th_percentile": 9, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "699c9baf3d29", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:57184", "tag.serverName": "699c9baf3d29,46197,1733217914195", "tag.clusterId": "b9c76278-f69a-4259-b357-0b850c15c1c5", "tag.Context": "regionserver", "tag.Hostname": "699c9baf3d29", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733217914195, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T09:25:30,422 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=33521 {}] master.MasterRpcServices(700): 699c9baf3d29,40779,1733217914054 reported a fatal error: ***** ABORTING region server 699c9baf3d29,40779,1733217914054: testing ***** 2024-12-03T09:25:30,423 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '699c9baf3d29,40779,1733217914054' ***** 2024-12-03T09:25:30,424 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T09:25:30,424 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T09:25:30,424 INFO [RS:0;699c9baf3d29:40779 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T09:25:30,424 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T09:25:30,424 INFO [RS:0;699c9baf3d29:40779 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T09:25:30,424 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(3091): Received CLOSE for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,424 INFO [regionserver/699c9baf3d29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:30,425 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(956): aborting server 699c9baf3d29,40779,1733217914054 2024-12-03T09:25:30,425 INFO [RS:0;699c9baf3d29:40779 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:25:30,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40727 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.2:51582 deadline: 1733217990424, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40779 startCode=1733217914054. As of locationSeqNum=12. 2024-12-03T09:25:30,425 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:30,425 INFO [RS:0;699c9baf3d29:40779 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;699c9baf3d29:40779. 2024-12-03T09:25:30,425 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,425 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,425 DEBUG [RS:0;699c9baf3d29:40779 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:25:30,425 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:30,425 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,425 DEBUG [RS:0;699c9baf3d29:40779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:30,425 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40727,1733217914150, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40727,1733217914150, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40779 startCode=1733217914054. As of locationSeqNum=12. 2024-12-03T09:25:30,426 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40727,1733217914150, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40779 startCode=1733217914054. As of locationSeqNum=12. 2024-12-03T09:25:30,426 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40727,1733217914150, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=699c9baf3d29 port=40779 startCode=1733217914054. As of locationSeqNum=12. 2024-12-03T09:25:30,426 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T09:25:30,426 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T09:25:30,426 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T09:25:30,426 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T09:25:30,426 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T09:25:30,427 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T09:25:30,427 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1325): Online Regions={60a49919b1e567cff342967e23c2879b=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., 1588230740=hbase:meta,,1.1588230740} 2024-12-03T09:25:30,427 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T09:25:30,427 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T09:25:30,427 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T09:25:30,427 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T09:25:30,427 DEBUG [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:30,427 ERROR [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-12-03T09:25:30,427 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T09:25:30,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:30,428 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T09:25:30,428 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,428 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733217930426Running coprocessor pre-close hooks at 1733217930426Disabling compacts and flushes for region at 1733217930426Disabling writes for close at 1733217930427 (+1 ms)Writing region close event to WAL at 1733217930427Running coprocessor post-close hooks at 1733217930427Closed at 1733217930428 (+1 ms) 2024-12-03T09:25:30,428 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217930425Running coprocessor pre-close hooks at 1733217930425Disabling compacts and flushes for region at 1733217930425Disabling writes for close at 1733217930425Writing region close event to WAL at 1733217930428 (+3 ms)Running coprocessor post-close hooks at 1733217930428Closed at 1733217930428 2024-12-03T09:25:30,428 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:30,428 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T09:25:30,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting 2024-12-03T09:25:30,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting 2024-12-03T09:25:30,429 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 from cache 2024-12-03T09:25:30,538 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:30,540 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1] 2024-12-03T09:25:30,542 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:30,544 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting 2024-12-03T09:25:30,544 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 699c9baf3d29,40779,1733217914054 aborting 2024-12-03T09:25:30,544 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 from cache 2024-12-03T09:25:30,571 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T09:25:30,571 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T09:25:30,627 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(976): stopping server 699c9baf3d29,40779,1733217914054; all regions closed. 2024-12-03T09:25:30,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741836_1012 (size=3561) 2024-12-03T09:25:30,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741836_1012 (size=3561) 2024-12-03T09:25:30,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741836_1012 (size=3561) 2024-12-03T09:25:30,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741834_1010 (size=1406) 2024-12-03T09:25:30,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741834_1010 (size=1406) 2024-12-03T09:25:30,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741834_1010 (size=1406) 2024-12-03T09:25:30,640 DEBUG [RS:0;699c9baf3d29:40779 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:30,640 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:30,640 INFO [RS:0;699c9baf3d29:40779 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:25:30,640 INFO [RS:0;699c9baf3d29:40779 {}] hbase.ChoreService(370): Chore service for: regionserver/699c9baf3d29:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T09:25:30,640 INFO [RS:0;699c9baf3d29:40779 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:25:30,641 INFO [regionserver/699c9baf3d29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:25:30,641 INFO [RS:0;699c9baf3d29:40779 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40779 2024-12-03T09:25:30,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:25:30,741 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/699c9baf3d29,40779,1733217914054 2024-12-03T09:25:30,742 INFO [RS:0;699c9baf3d29:40779 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:25:30,748 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:30,750 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1] 2024-12-03T09:25:30,752 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 699c9baf3d29:40779 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 699c9baf3d29/172.17.0.2:40779 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:30,753 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [699c9baf3d29,40779,1733217914054] 2024-12-03T09:25:30,753 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1, error=java.net.ConnectException: Call to address=699c9baf3d29:40779 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 699c9baf3d29/172.17.0.2:40779 2024-12-03T09:25:30,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 is java.net.ConnectException: Connection refused 2024-12-03T09:25:30,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 from cache 2024-12-03T09:25:30,754 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 699c9baf3d29:40779 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 699c9baf3d29/172.17.0.2:40779 2024-12-03T09:25:30,763 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/699c9baf3d29,40779,1733217914054 already deleted, retry=false 2024-12-03T09:25:30,763 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 699c9baf3d29,40779,1733217914054 on 699c9baf3d29,33521,1733217913369 2024-12-03T09:25:30,767 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 699c9baf3d29,40779,1733217914054, splitWal=true, meta=true 2024-12-03T09:25:30,769 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for 699c9baf3d29,40779,1733217914054 (carryingMeta=true) 699c9baf3d29,40779,1733217914054/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@7084729f[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T09:25:30,770 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 699c9baf3d29,40779,1733217914054, splitWal=true, meta=true 2024-12-03T09:25:30,771 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 699c9baf3d29,40779,1733217914054, splitWal=true, meta=true, isMeta: true 2024-12-03T09:25:30,772 DEBUG [PEWorker-1 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting 2024-12-03T09:25:30,773 INFO [PEWorker-1 {}] master.SplitWALManager(105): 699c9baf3d29,40779,1733217914054 WAL count=1, meta=true 2024-12-03T09:25:30,775 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta}] 2024-12-03T09:25:30,779 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=699c9baf3d29,46197,1733217914195 2024-12-03T09:25:30,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, worker=699c9baf3d29,46197,1733217914195}] 2024-12-03T09:25:30,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:30,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40779-0x1019a8282cb0001, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:30,854 INFO [RS:0;699c9baf3d29:40779 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:25:30,855 INFO [RS:0;699c9baf3d29:40779 {}] regionserver.HRegionServer(1031): Exiting; stopping=699c9baf3d29,40779,1733217914054; zookeeper connection closed. 2024-12-03T09:25:30,855 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@77ccf324 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@77ccf324 2024-12-03T09:25:30,944 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T09:25:30,945 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44679, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T09:25:30,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46197 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T09:25:30,963 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, size=3.5 K (3561bytes) 2024-12-03T09:25:30,963 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta 2024-12-03T09:25:30,964 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta after 1ms 2024-12-03T09:25:30,967 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:30,967 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta took 4ms 2024-12-03T09:25:30,974 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-12-03T09:25:30,975 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta so closing down 2024-12-03T09:25:30,976 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:30,976 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp 2024-12-03T09:25:30,978 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp 2024-12-03T09:25:30,979 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741892_1070 (size=3346) 2024-12-03T09:25:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741892_1070 (size=3346) 2024-12-03T09:25:30,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741892_1070 (size=3346) 2024-12-03T09:25:30,990 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:30,991 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T09:25:30,992 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 25 ms; skipped=1; WAL=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false 2024-12-03T09:25:30,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, journal: Splitting hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, size=3.5 K (3561bytes) at 1733217930963Finishing writing output for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta so closing down at 1733217930975 (+12 ms)Creating recovered edits writer path=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp at 1733217930978 (+3 ms)3 split writer threads finished at 1733217930979 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733217930990 (+11 ms)Rename recovered edits hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta.temp to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1733217930992 (+2 ms)Processed 16 edits across 1 Regions in 25 ms; skipped=1; WAL=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, size=3.5 K, length=3561, corrupted=false, cancelled=false at 1733217930992 2024-12-03T09:25:30,992 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta 2024-12-03T09:25:30,993 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T09:25:30,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T09:25:31,001 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs 2024-12-03T09:25:31,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T09:25:31,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, worker=699c9baf3d29,46197,1733217914195 in 220 msec 2024-12-03T09:25:31,006 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=699c9baf3d29,46197,1733217914195 2024-12-03T09:25:31,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T09:25:31,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 699c9baf3d29%2C40779%2C1733217914054.meta.1733217916136.meta, worker=699c9baf3d29,46197,1733217914195 in 233 msec 2024-12-03T09:25:31,012 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting dir is empty, no logs to split. 2024-12-03T09:25:31,012 INFO [PEWorker-2 {}] master.SplitWALManager(105): 699c9baf3d29,40779,1733217914054 WAL count=0, meta=true 2024-12-03T09:25:31,012 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 699c9baf3d29,40779,1733217914054 WAL splitting is done? wals=0, meta=true 2024-12-03T09:25:31,014 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T09:25:31,016 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:25:31,017 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T09:25:31,057 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:31,059 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1] 2024-12-03T09:25:31,059 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 699c9baf3d29:40779 this server is in the failed servers list 2024-12-03T09:25:31,060 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=699c9baf3d29:40779 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,060 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,060 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 from cache 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(204): Hosts are {699c9baf3d29=0} racks are {/default-rack=0} 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T09:25:31,168 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T09:25:31,168 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T09:25:31,168 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T09:25:31,169 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:31,170 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,40727,1733217914150, state=OPENING 2024-12-03T09:25:31,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,226 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,226 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T09:25:31,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,40727,1733217914150}] 2024-12-03T09:25:31,226 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,384 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T09:25:31,385 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:31,385 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T09:25:31,388 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=699c9baf3d29%2C40727%2C1733217914150.meta, suffix=.meta, logDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150, archiveDir=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs, maxLogs=32 2024-12-03T09:25:31,400 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150/699c9baf3d29%2C40727%2C1733217914150.meta.1733217931389.meta, exclude list is [], retry=0 2024-12-03T09:25:31,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:31,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:31,403 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:31,405 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150/699c9baf3d29%2C40727%2C1733217914150.meta.1733217931389.meta 2024-12-03T09:25:31,406 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:31,406 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:31,407 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T09:25:31,407 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T09:25:31,407 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T09:25:31,407 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T09:25:31,407 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:31,408 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T09:25:31,408 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T09:25:31,409 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T09:25:31,410 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T09:25:31,410 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:31,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:31,411 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T09:25:31,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T09:25:31,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:31,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:31,412 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T09:25:31,413 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T09:25:31,413 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:31,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:31,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T09:25:31,414 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T09:25:31,414 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:31,415 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T09:25:31,415 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T09:25:31,416 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:31,417 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740 2024-12-03T09:25:31,418 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T09:25:31,420 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:31,422 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T09:25:31,422 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-12-03T09:25:31,439 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/cd9f66b897674afe863dfdacb116db80 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b./info:regioninfo/1733217930358/Put/seqid=0 2024-12-03T09:25:31,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741894_1072 (size=11177) 2024-12-03T09:25:31,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741894_1072 (size=11177) 2024-12-03T09:25:31,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741894_1072 (size=11177) 2024-12-03T09:25:31,446 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/cd9f66b897674afe863dfdacb116db80 2024-12-03T09:25:31,464 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/ns/a1427f8ceca04cb9bbc13fe7852b5273 is 43, key is default/ns:d/1733217916284/Put/seqid=0 2024-12-03T09:25:31,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741895_1073 (size=5153) 2024-12-03T09:25:31,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741895_1073 (size=5153) 2024-12-03T09:25:31,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741895_1073 (size=5153) 2024-12-03T09:25:31,471 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/ns/a1427f8ceca04cb9bbc13fe7852b5273 2024-12-03T09:25:31,490 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/table/f60ed72e81524d4d80f890b9099f16a7 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733217928622/Put/seqid=0 2024-12-03T09:25:31,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741896_1074 (size=5431) 2024-12-03T09:25:31,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741896_1074 (size=5431) 2024-12-03T09:25:31,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741896_1074 (size=5431) 2024-12-03T09:25:31,496 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/table/f60ed72e81524d4d80f890b9099f16a7 2024-12-03T09:25:31,503 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/cd9f66b897674afe863dfdacb116db80 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/info/cd9f66b897674afe863dfdacb116db80 2024-12-03T09:25:31,510 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/info/cd9f66b897674afe863dfdacb116db80, entries=36, sequenceid=18, filesize=10.9 K 2024-12-03T09:25:31,512 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/ns/a1427f8ceca04cb9bbc13fe7852b5273 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/ns/a1427f8ceca04cb9bbc13fe7852b5273 2024-12-03T09:25:31,518 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/ns/a1427f8ceca04cb9bbc13fe7852b5273, entries=2, sequenceid=18, filesize=5.0 K 2024-12-03T09:25:31,519 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/table/f60ed72e81524d4d80f890b9099f16a7 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/table/f60ed72e81524d4d80f890b9099f16a7 2024-12-03T09:25:31,525 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/table/f60ed72e81524d4d80f890b9099f16a7, entries=2, sequenceid=18, filesize=5.3 K 2024-12-03T09:25:31,526 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=18, compaction requested=false; wal=null 2024-12-03T09:25:31,527 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-12-03T09:25:31,528 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T09:25:31,528 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T09:25:31,529 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T09:25:31,531 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T09:25:31,533 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-12-03T09:25:31,534 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70473238, jitterRate=0.050133079290390015}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T09:25:31,535 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T09:25:31,535 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733217931408Writing region info on filesystem at 1733217931408Initializing all the Stores at 1733217931409 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217931409Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217931409Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217931409Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733217931409Obtaining lock to block concurrent updates at 1733217931422 (+13 ms)Preparing flush snapshotting stores in 1588230740 at 1733217931422Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1733217931422Flushing stores of hbase:meta,,1.1588230740 at 1733217931422Flushing 1588230740/info: creating writer at 1733217931423 (+1 ms)Flushing 1588230740/info: appending metadata at 1733217931439 (+16 ms)Flushing 1588230740/info: closing flushed file at 1733217931439Flushing 1588230740/ns: creating writer at 1733217931451 (+12 ms)Flushing 1588230740/ns: appending metadata at 1733217931464 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1733217931464Flushing 1588230740/table: creating writer at 1733217931476 (+12 ms)Flushing 1588230740/table: appending metadata at 1733217931489 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733217931489Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3285c104: reopening flushed file at 1733217931502 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3897b60a: reopening flushed file at 1733217931511 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@9482c52: reopening flushed file at 1733217931518 (+7 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 103ms, sequenceid=18, compaction requested=false; wal=null at 1733217931526 (+8 ms)Cleaning up temporary data from old regions at 1733217931528 (+2 ms)Running coprocessor post-open hooks at 1733217931535 (+7 ms)Region opened successfully at 1733217931535 2024-12-03T09:25:31,536 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1733217931378 2024-12-03T09:25:31,538 DEBUG [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T09:25:31,538 INFO [RS_OPEN_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T09:25:31,539 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:31,540 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 699c9baf3d29,40727,1733217914150, state=OPEN 2024-12-03T09:25:31,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T09:25:31,552 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:31,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,552 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T09:25:31,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T09:25:31,555 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=699c9baf3d29,40727,1733217914150 in 326 msec 2024-12-03T09:25:31,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T09:25:31,558 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 541 msec 2024-12-03T09:25:31,558 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 699c9baf3d29,40779,1733217914054 had 2 regions 2024-12-03T09:25:31,559 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 699c9baf3d29,40779,1733217914054, splitWal=true, meta=true, isMeta: false 2024-12-03T09:25:31,561 INFO [PEWorker-1 {}] master.SplitWALManager(105): 699c9baf3d29,40779,1733217914054 WAL count=1, meta=false 2024-12-03T09:25:31,561 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 699c9baf3d29%2C40779%2C1733217914054.1733217915683}] 2024-12-03T09:25:31,563 DEBUG [PEWorker-3 {}] master.SplitWALManager(158): Acquired split WAL worker=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:31,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 699c9baf3d29%2C40779%2C1733217914054.1733217915683, worker=699c9baf3d29,40727,1733217914150}] 2024-12-03T09:25:31,567 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:31,568 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40727,1733217914150, seqNum=-1] 2024-12-03T09:25:31,688 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 699c9baf3d29:40779 this server is in the failed servers list 2024-12-03T09:25:31,688 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=699c9baf3d29:40779 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,688 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,689 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=12 from cache 2024-12-03T09:25:31,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40727 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-12-03T09:25:31,737 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683, size=1.4 K (1406bytes) 2024-12-03T09:25:31,737 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 2024-12-03T09:25:31,738 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 after 1ms 2024-12-03T09:25:31,740 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:31,740 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 took 3ms 2024-12-03T09:25:31,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for 60a49919b1e567cff342967e23c2879b: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T09:25:31,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 so closing down 2024-12-03T09:25:31,743 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:31,743 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:31,743 INFO [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683, size=1.4 K, length=1406, corrupted=false, cancelled=false 2024-12-03T09:25:31,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683, journal: Splitting hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683, size=1.4 K (1406bytes) at 1733217931737Finishing writing output for hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 so closing down at 1733217931743 (+6 ms)3 split writer threads finished at 1733217931743Processed 6 edits across 0 Regions in 2 ms; skipped=6; WAL=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683, size=1.4 K, length=1406, corrupted=false, cancelled=false at 1733217931744 (+1 ms) 2024-12-03T09:25:31,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 2024-12-03T09:25:31,744 DEBUG [RS_LOG_REPLAY_OPS-regionserver/699c9baf3d29:0-0 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-12-03T09:25:31,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=33521 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-12-03T09:25:31,747 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting/699c9baf3d29%2C40779%2C1733217914054.1733217915683 to hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs 2024-12-03T09:25:31,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-12-03T09:25:31,750 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 699c9baf3d29%2C40779%2C1733217914054.1733217915683, worker=699c9baf3d29,40727,1733217914150 in 184 msec 2024-12-03T09:25:31,751 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=699c9baf3d29,40727,1733217914150 2024-12-03T09:25:31,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-12-03T09:25:31,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 699c9baf3d29%2C40779%2C1733217914054.1733217915683, worker=699c9baf3d29,40727,1733217914150 in 190 msec 2024-12-03T09:25:31,755 INFO [PEWorker-1 {}] master.SplitLogManager(171): hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40779,1733217914054-splitting dir is empty, no logs to split. 2024-12-03T09:25:31,755 INFO [PEWorker-1 {}] master.SplitWALManager(105): 699c9baf3d29,40779,1733217914054 WAL count=0, meta=false 2024-12-03T09:25:31,755 DEBUG [PEWorker-1 {}] procedure.ServerCrashProcedure(329): Check if 699c9baf3d29,40779,1733217914054 WAL splitting is done? wals=0, meta=false 2024-12-03T09:25:31,758 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN}] 2024-12-03T09:25:31,759 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN 2024-12-03T09:25:31,761 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T09:25:31,905 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=18] 2024-12-03T09:25:31,905 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 699c9baf3d29:40779 this server is in the failed servers list 2024-12-03T09:25:31,905 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=699c9baf3d29:40779 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,906 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 699c9baf3d29:40779 2024-12-03T09:25:31,906 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,40779,1733217914054, seqNum=18 from cache 2024-12-03T09:25:31,911 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(204): Hosts are {699c9baf3d29=0} racks are {/default-rack=0} 2024-12-03T09:25:31,911 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T09:25:31,911 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T09:25:31,911 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T09:25:31,911 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T09:25:31,912 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T09:25:31,912 INFO [699c9baf3d29:33521 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T09:25:31,912 DEBUG [699c9baf3d29:33521 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T09:25:31,912 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPENING, regionLocation=699c9baf3d29,46197,1733217914195 2024-12-03T09:25:31,913 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 699c9baf3d29:40779 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 699c9baf3d29/172.17.0.2:40779 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:31,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1, error=java.net.ConnectException: Call to address=699c9baf3d29:40779 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 699c9baf3d29/172.17.0.2:40779 2024-12-03T09:25:31,913 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-12-03T09:25:31,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40779,1733217914054, seqNum=-1 from cache 2024-12-03T09:25:31,914 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.FailedServers(52): Added failed server with address 699c9baf3d29:40779 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 699c9baf3d29/172.17.0.2:40779 2024-12-03T09:25:32,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T09:25:32,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=699c9baf3d29,40727,1733217914150, seqNum=-1] 2024-12-03T09:25:32,018 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:25:32,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:46239, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:25:32,023 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN because future has completed 2024-12-03T09:25:32,024 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,46197,1733217914195}] 2024-12-03T09:25:32,182 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:32,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => 60a49919b1e567cff342967e23c2879b, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:32,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:32,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,182 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,184 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,185 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf1 2024-12-03T09:25:32,185 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,192 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf1/a1a14405012a47e099b2a7a58bcd2a37 2024-12-03T09:25:32,192 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,192 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,193 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 60a49919b1e567cff342967e23c2879b columnFamilyName cf2 2024-12-03T09:25:32,193 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,200 DEBUG [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/cf2/4fe56521b1d44b1faa6e2cd8688b3a84 2024-12-03T09:25:32,200 INFO [StoreOpener-60a49919b1e567cff342967e23c2879b-1 {}] regionserver.HStore(327): Store=60a49919b1e567cff342967e23c2879b/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,201 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,201 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,202 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,203 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,203 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,204 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T09:25:32,205 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,206 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened 60a49919b1e567cff342967e23c2879b; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72822710, jitterRate=0.08514294028282166}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T09:25:32,206 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:32,206 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for 60a49919b1e567cff342967e23c2879b: Running coprocessor pre-open hook at 1733217932183Writing region info on filesystem at 1733217932183Initializing all the Stores at 1733217932184 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932184Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932184Cleaning up temporary data from old regions at 1733217932203 (+19 ms)Running coprocessor post-open hooks at 1733217932206 (+3 ms)Region opened successfully at 1733217932206 2024-12-03T09:25:32,207 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., pid=21, masterSystemTime=1733217932177 2024-12-03T09:25:32,210 DEBUG [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:32,210 INFO [RS_OPEN_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:32,211 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=60a49919b1e567cff342967e23c2879b, regionState=OPEN, openSeqNum=18, regionLocation=699c9baf3d29,46197,1733217914195 2024-12-03T09:25:32,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,46197,1733217914195 because future has completed 2024-12-03T09:25:32,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-12-03T09:25:32,217 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure 60a49919b1e567cff342967e23c2879b, server=699c9baf3d29,46197,1733217914195 in 191 msec 2024-12-03T09:25:32,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-12-03T09:25:32,220 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 699c9baf3d29,40779,1733217914054 after splitting done 2024-12-03T09:25:32,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=60a49919b1e567cff342967e23c2879b, ASSIGN in 459 msec 2024-12-03T09:25:32,222 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b., hostname=699c9baf3d29,46197,1733217914195, seqNum=18] 2024-12-03T09:25:32,223 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T09:25:32,223 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 699c9baf3d29,40779,1733217914054, splitWal=true, meta=true in 1.4560 sec 2024-12-03T09:25:32,224 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44584, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T09:25:32,239 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterRegionMovedWithMultiCF Thread=414 (was 418), OpenFileDescriptor=1052 (was 1027) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 199) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=13136 (was 13168) 2024-12-03T09:25:32,239 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-12-03T09:25:32,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=414, OpenFileDescriptor=1052, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=13134 2024-12-03T09:25:32,251 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1052 is superior to 1024 2024-12-03T09:25:32,264 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:32,265 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:32,266 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:32,269 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-05201907, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-05201907, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:32,281 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-05201907/hregion-05201907.1733217932269, exclude list is [], retry=0 2024-12-03T09:25:32,283 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:32,283 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:32,284 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:32,287 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-05201907/hregion-05201907.1733217932269 2024-12-03T09:25:32,287 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:32,287 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 47ec9ac36290d3327d19f02721e24a9e, NAME => 'testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:32,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741898_1076 (size=67) 2024-12-03T09:25:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741898_1076 (size=67) 2024-12-03T09:25:32,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741898_1076 (size=67) 2024-12-03T09:25:32,296 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:32,297 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,299 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName a 2024-12-03T09:25:32,299 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,299 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,299 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,300 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName b 2024-12-03T09:25:32,300 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,301 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,301 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,302 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName c 2024-12-03T09:25:32,302 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,302 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,303 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,303 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,303 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,305 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,305 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,305 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:32,306 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,308 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:32,309 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ec9ac36290d3327d19f02721e24a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62660422, jitterRate=-0.06628695130348206}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:32,309 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ec9ac36290d3327d19f02721e24a9e: Writing region info on filesystem at 1733217932296Initializing all the Stores at 1733217932297 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932297Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932297Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932297Cleaning up temporary data from old regions at 1733217932305 (+8 ms)Region opened successfully at 1733217932309 (+4 ms) 2024-12-03T09:25:32,309 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47ec9ac36290d3327d19f02721e24a9e, disabling compactions & flushes 2024-12-03T09:25:32,310 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,310 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,310 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. after waiting 0 ms 2024-12-03T09:25:32,310 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,310 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,310 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47ec9ac36290d3327d19f02721e24a9e: Waiting for close lock at 1733217932309Disabling compacts and flushes for region at 1733217932309Disabling writes for close at 1733217932310 (+1 ms)Writing region close event to WAL at 1733217932310Closed at 1733217932310 2024-12-03T09:25:32,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741897_1075 (size=95) 2024-12-03T09:25:32,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741897_1075 (size=95) 2024-12-03T09:25:32,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741897_1075 (size=95) 2024-12-03T09:25:32,316 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:32,316 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-05201907:(num 1733217932269) 2024-12-03T09:25:32,316 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:32,318 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:32,330 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, exclude list is [], retry=0 2024-12-03T09:25:32,333 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:32,334 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:32,334 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:32,336 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 2024-12-03T09:25:32,336 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:32,336 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47ec9ac36290d3327d19f02721e24a9e, NAME => 'testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:32,337 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:32,337 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,337 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,340 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,341 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName a 2024-12-03T09:25:32,341 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,342 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,342 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,343 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName b 2024-12-03T09:25:32,343 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,343 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,343 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,344 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName c 2024-12-03T09:25:32,344 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,344 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,345 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,345 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,346 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,347 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,347 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,347 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:32,349 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,350 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ec9ac36290d3327d19f02721e24a9e; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61207144, jitterRate=-0.08794248104095459}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:32,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ec9ac36290d3327d19f02721e24a9e: Writing region info on filesystem at 1733217932337Initializing all the Stores at 1733217932338 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932338Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932340 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932340Cleaning up temporary data from old regions at 1733217932347 (+7 ms)Region opened successfully at 1733217932350 (+3 ms) 2024-12-03T09:25:32,376 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47ec9ac36290d3327d19f02721e24a9e 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T09:25:32,391 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/a/cddf693847814105ae0768ae2fe0101a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733217932350/Put/seqid=0 2024-12-03T09:25:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741900_1078 (size=5958) 2024-12-03T09:25:32,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741900_1078 (size=5958) 2024-12-03T09:25:32,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741900_1078 (size=5958) 2024-12-03T09:25:32,398 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/a/cddf693847814105ae0768ae2fe0101a 2024-12-03T09:25:32,417 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/8d7333a5d15345639550a207eee28d9e is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733217932358/Put/seqid=0 2024-12-03T09:25:32,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741901_1079 (size=5958) 2024-12-03T09:25:32,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741901_1079 (size=5958) 2024-12-03T09:25:32,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741901_1079 (size=5958) 2024-12-03T09:25:32,826 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/8d7333a5d15345639550a207eee28d9e 2024-12-03T09:25:32,849 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/c/4bff8f5f8d864a4e920ecfdfd5cace8d is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733217932367/Put/seqid=0 2024-12-03T09:25:32,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741902_1080 (size=5958) 2024-12-03T09:25:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741902_1080 (size=5958) 2024-12-03T09:25:32,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741902_1080 (size=5958) 2024-12-03T09:25:32,855 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/c/4bff8f5f8d864a4e920ecfdfd5cace8d 2024-12-03T09:25:32,860 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/a/cddf693847814105ae0768ae2fe0101a as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/a/cddf693847814105ae0768ae2fe0101a 2024-12-03T09:25:32,865 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/a/cddf693847814105ae0768ae2fe0101a, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T09:25:32,866 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/8d7333a5d15345639550a207eee28d9e as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/8d7333a5d15345639550a207eee28d9e 2024-12-03T09:25:32,871 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/8d7333a5d15345639550a207eee28d9e, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T09:25:32,872 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/c/4bff8f5f8d864a4e920ecfdfd5cace8d as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/c/4bff8f5f8d864a4e920ecfdfd5cace8d 2024-12-03T09:25:32,877 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/c/4bff8f5f8d864a4e920ecfdfd5cace8d, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T09:25:32,878 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 47ec9ac36290d3327d19f02721e24a9e in 502ms, sequenceid=33, compaction requested=false 2024-12-03T09:25:32,878 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 47ec9ac36290d3327d19f02721e24a9e: 2024-12-03T09:25:32,878 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 47ec9ac36290d3327d19f02721e24a9e, disabling compactions & flushes 2024-12-03T09:25:32,878 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,879 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,879 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. after waiting 0 ms 2024-12-03T09:25:32,879 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,880 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. 2024-12-03T09:25:32,880 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 47ec9ac36290d3327d19f02721e24a9e: Waiting for close lock at 1733217932878Disabling compacts and flushes for region at 1733217932878Disabling writes for close at 1733217932879 (+1 ms)Writing region close event to WAL at 1733217932880 (+1 ms)Closed at 1733217932880 2024-12-03T09:25:32,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741899_1077 (size=3384) 2024-12-03T09:25:32,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741899_1077 (size=3384) 2024-12-03T09:25:32,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741899_1077 (size=3384) 2024-12-03T09:25:32,888 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/8d7333a5d15345639550a207eee28d9e to hdfs://localhost:37427/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/8d7333a5d15345639550a207eee28d9e 2024-12-03T09:25:32,902 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, size=3.3 K (3384bytes) 2024-12-03T09:25:32,902 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 2024-12-03T09:25:32,903 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 after 1ms 2024-12-03T09:25:32,905 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:32,905 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 took 3ms 2024-12-03T09:25:32,907 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 so closing down 2024-12-03T09:25:32,907 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:32,908 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733217932319.temp 2024-12-03T09:25:32,909 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp 2024-12-03T09:25:32,909 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741903_1081 (size=2944) 2024-12-03T09:25:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741903_1081 (size=2944) 2024-12-03T09:25:32,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741903_1081 (size=2944) 2024-12-03T09:25:32,916 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:32,917 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 2024-12-03T09:25:32,917 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, size=3.3 K, length=3384, corrupted=false, cancelled=false 2024-12-03T09:25:32,917 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, journal: Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, size=3.3 K (3384bytes) at 1733217932902Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 so closing down at 1733217932907 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp at 1733217932909 (+2 ms)3 split writer threads finished at 1733217932909Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733217932916 (+7 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000003-wal.1733217932319.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 at 1733217932917 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319, size=3.3 K, length=3384, corrupted=false, cancelled=false at 1733217932917 2024-12-03T09:25:32,918 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932319 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217932319 2024-12-03T09:25:32,919 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 2024-12-03T09:25:32,919 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:32,921 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:32,933 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932921, exclude list is [], retry=0 2024-12-03T09:25:32,936 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:32,936 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:32,936 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:32,939 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932921 2024-12-03T09:25:32,939 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:32,939 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 47ec9ac36290d3327d19f02721e24a9e, NAME => 'testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:32,939 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:32,939 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,940 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,941 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,942 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName a 2024-12-03T09:25:32,942 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,948 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/a/cddf693847814105ae0768ae2fe0101a 2024-12-03T09:25:32,948 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,949 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,951 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName b 2024-12-03T09:25:32,951 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,951 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,952 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,953 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 47ec9ac36290d3327d19f02721e24a9e columnFamilyName c 2024-12-03T09:25:32,953 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:32,960 DEBUG [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/c/4bff8f5f8d864a4e920ecfdfd5cace8d 2024-12-03T09:25:32,960 INFO [StoreOpener-47ec9ac36290d3327d19f02721e24a9e-1 {}] regionserver.HStore(327): Store=47ec9ac36290d3327d19f02721e24a9e/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:32,960 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,962 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:32,963 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 2024-12-03T09:25:32,965 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:32,967 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 2024-12-03T09:25:32,967 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 47ec9ac36290d3327d19f02721e24a9e 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T09:25:32,980 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/a4cab5d6b8e84f0fa41ae8f4935e0bd5 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733217932358/Put/seqid=0 2024-12-03T09:25:32,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741905_1083 (size=5958) 2024-12-03T09:25:32,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741905_1083 (size=5958) 2024-12-03T09:25:32,987 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/a4cab5d6b8e84f0fa41ae8f4935e0bd5 2024-12-03T09:25:32,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741905_1083 (size=5958) 2024-12-03T09:25:32,994 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/.tmp/b/a4cab5d6b8e84f0fa41ae8f4935e0bd5 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/a4cab5d6b8e84f0fa41ae8f4935e0bd5 2024-12-03T09:25:33,000 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/b/a4cab5d6b8e84f0fa41ae8f4935e0bd5, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T09:25:33,000 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 47ec9ac36290d3327d19f02721e24a9e in 33ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T09:25:33,001 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/0000000000000000032 2024-12-03T09:25:33,002 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:33,002 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:33,003 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:33,004 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 47ec9ac36290d3327d19f02721e24a9e 2024-12-03T09:25:33,007 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/47ec9ac36290d3327d19f02721e24a9e/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T09:25:33,008 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 47ec9ac36290d3327d19f02721e24a9e; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67120467, jitterRate=1.7289817333221436E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:33,008 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 47ec9ac36290d3327d19f02721e24a9e: Writing region info on filesystem at 1733217932940Initializing all the Stores at 1733217932941 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932941Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932941Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217932941Obtaining lock to block concurrent updates at 1733217932967 (+26 ms)Preparing flush snapshotting stores in 47ec9ac36290d3327d19f02721e24a9e at 1733217932967Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733217932967Flushing stores of testReplayEditsWrittenViaHRegion,,1733217932264.47ec9ac36290d3327d19f02721e24a9e. at 1733217932967Flushing 47ec9ac36290d3327d19f02721e24a9e/b: creating writer at 1733217932967Flushing 47ec9ac36290d3327d19f02721e24a9e/b: appending metadata at 1733217932979 (+12 ms)Flushing 47ec9ac36290d3327d19f02721e24a9e/b: closing flushed file at 1733217932979Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5902699f: reopening flushed file at 1733217932993 (+14 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 47ec9ac36290d3327d19f02721e24a9e in 33ms, sequenceid=32, compaction requested=false; wal=null at 1733217933000 (+7 ms)Cleaning up temporary data from old regions at 1733217933002 (+2 ms)Region opened successfully at 1733217933008 (+6 ms) 2024-12-03T09:25:33,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterPartialFlush Thread=422 (was 414) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:52206 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:40660 [Waiting for operation #43] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:39626 [Waiting for operation #38] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:58800 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:50420 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46484 [Waiting for operation #42] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1126 (was 1052) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=207 (was 207), ProcessCount=11 (was 11), AvailableMemoryMB=13127 (was 13134) 2024-12-03T09:25:33,031 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-12-03T09:25:33,043 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=422, OpenFileDescriptor=1126, MaxFileDescriptor=1048576, SystemLoadAverage=207, ProcessCount=11, AvailableMemoryMB=13126 2024-12-03T09:25:33,043 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1126 is superior to 1024 2024-12-03T09:25:33,055 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:33,057 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:33,057 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:33,059 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-50930892, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-50930892, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:33,069 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-50930892/hregion-50930892.1733217933059, exclude list is [], retry=0 2024-12-03T09:25:33,072 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:33,072 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:33,072 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:33,074 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-50930892/hregion-50930892.1733217933059 2024-12-03T09:25:33,074 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:33,075 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 40fdae4b1a62d6d08dd8d623d21eb5c9, NAME => 'testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:33,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741907_1085 (size=68) 2024-12-03T09:25:33,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741907_1085 (size=68) 2024-12-03T09:25:33,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741907_1085 (size=68) 2024-12-03T09:25:33,083 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:33,084 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,085 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName a 2024-12-03T09:25:33,085 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,086 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,086 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,087 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName b 2024-12-03T09:25:33,087 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,087 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,087 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,088 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName c 2024-12-03T09:25:33,088 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,088 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,089 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,089 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,090 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,090 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,091 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,091 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:33,093 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,095 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:33,096 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40fdae4b1a62d6d08dd8d623d21eb5c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61032102, jitterRate=-0.09055081009864807}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:33,097 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: Writing region info on filesystem at 1733217933083Initializing all the Stores at 1733217933084 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933084Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933084Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933084Cleaning up temporary data from old regions at 1733217933091 (+7 ms)Region opened successfully at 1733217933097 (+6 ms) 2024-12-03T09:25:33,097 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40fdae4b1a62d6d08dd8d623d21eb5c9, disabling compactions & flushes 2024-12-03T09:25:33,097 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:33,097 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:33,097 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. after waiting 0 ms 2024-12-03T09:25:33,097 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:33,098 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:33,098 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: Waiting for close lock at 1733217933097Disabling compacts and flushes for region at 1733217933097Disabling writes for close at 1733217933097Writing region close event to WAL at 1733217933098 (+1 ms)Closed at 1733217933098 2024-12-03T09:25:33,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741906_1084 (size=95) 2024-12-03T09:25:33,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741906_1084 (size=95) 2024-12-03T09:25:33,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741906_1084 (size=95) 2024-12-03T09:25:33,104 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:33,104 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-50930892:(num 1733217933059) 2024-12-03T09:25:33,104 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:33,106 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:33,117 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, exclude list is [], retry=0 2024-12-03T09:25:33,119 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:33,119 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:33,120 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:33,121 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 2024-12-03T09:25:33,121 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:33,175 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40fdae4b1a62d6d08dd8d623d21eb5c9, NAME => 'testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:33,177 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,177 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:33,177 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,177 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,179 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,180 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName a 2024-12-03T09:25:33,180 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,180 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,181 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,181 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName b 2024-12-03T09:25:33,181 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,182 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,182 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,183 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName c 2024-12-03T09:25:33,183 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:33,183 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:33,184 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,184 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,186 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,187 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,187 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,187 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:33,188 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,189 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40fdae4b1a62d6d08dd8d623d21eb5c9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64649253, jitterRate=-0.03665105998516083}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:33,189 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:33,190 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: Running coprocessor pre-open hook at 1733217933177Writing region info on filesystem at 1733217933177Initializing all the Stores at 1733217933178 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933178Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933178Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217933178Cleaning up temporary data from old regions at 1733217933187 (+9 ms)Running coprocessor post-open hooks at 1733217933189 (+2 ms)Region opened successfully at 1733217933190 (+1 ms) 2024-12-03T09:25:33,201 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T09:25:33,203 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:34,131 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:25:34,169 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T09:25:34,169 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:34,170 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T09:25:34,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:34,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T09:25:34,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:34,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T09:25:34,172 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:34,203 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:35,204 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:36,205 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:37,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:38,207 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:39,208 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:40,210 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:41,212 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:42,213 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 40fdae4b1a62d6d08dd8d623d21eb5c9/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:42,216 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: 2024-12-03T09:25:42,217 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,230 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: 2024-12-03T09:25:42,230 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T09:25:42,230 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 40fdae4b1a62d6d08dd8d623d21eb5c9, disabling compactions & flushes 2024-12-03T09:25:42,230 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,230 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,231 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. after waiting 0 ms 2024-12-03T09:25:42,231 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,231 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,231 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. 2024-12-03T09:25:42,231 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: Waiting for close lock at 1733217942230Running coprocessor pre-close hooks at 1733217942230Disabling compacts and flushes for region at 1733217942230Disabling writes for close at 1733217942231 (+1 ms)Writing region close event to WAL at 1733217942231Running coprocessor post-close hooks at 1733217942231Closed at 1733217942231 2024-12-03T09:25:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741908_1086 (size=2691) 2024-12-03T09:25:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741908_1086 (size=2691) 2024-12-03T09:25:42,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741908_1086 (size=2691) 2024-12-03T09:25:42,249 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, size=2.6 K (2691bytes) 2024-12-03T09:25:42,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 2024-12-03T09:25:42,249 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 after 0ms 2024-12-03T09:25:42,252 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:42,252 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 took 4ms 2024-12-03T09:25:42,254 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 so closing down 2024-12-03T09:25:42,254 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:42,255 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733217933106.temp 2024-12-03T09:25:42,256 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp 2024-12-03T09:25:42,256 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:42,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741909_1087 (size=2094) 2024-12-03T09:25:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741909_1087 (size=2094) 2024-12-03T09:25:42,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741909_1087 (size=2094) 2024-12-03T09:25:42,262 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:42,263 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 2024-12-03T09:25:42,263 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, size=2.6 K, length=2691, corrupted=false, cancelled=false 2024-12-03T09:25:42,264 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, journal: Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, size=2.6 K (2691bytes) at 1733217942249Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 so closing down at 1733217942254 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp at 1733217942256 (+2 ms)3 split writer threads finished at 1733217942256Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733217942262 (+6 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000004-wal.1733217933106.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 at 1733217942263 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106, size=2.6 K, length=2691, corrupted=false, cancelled=false at 1733217942264 (+1 ms) 2024-12-03T09:25:42,265 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217933106 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217933106 2024-12-03T09:25:42,266 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 2024-12-03T09:25:42,267 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:42,268 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:42,280 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217942269, exclude list is [], retry=0 2024-12-03T09:25:42,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:42,283 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:42,284 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:42,285 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217942269 2024-12-03T09:25:42,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:42,286 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 40fdae4b1a62d6d08dd8d623d21eb5c9, NAME => 'testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:42,286 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,287 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:42,287 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,287 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,289 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,290 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName a 2024-12-03T09:25:42,290 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:42,291 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:42,291 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,292 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName b 2024-12-03T09:25:42,292 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:42,293 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:42,293 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,294 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 40fdae4b1a62d6d08dd8d623d21eb5c9 columnFamilyName c 2024-12-03T09:25:42,294 DEBUG [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:42,294 INFO [StoreOpener-40fdae4b1a62d6d08dd8d623d21eb5c9-1 {}] regionserver.HStore(327): Store=40fdae4b1a62d6d08dd8d623d21eb5c9/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:42,294 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,295 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,297 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,297 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 2024-12-03T09:25:42,299 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:42,301 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 2024-12-03T09:25:42,301 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T09:25:42,315 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/a/b19ca9a6c207443d8cb27ddc6c846b42 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733217942222/Put/seqid=0 2024-12-03T09:25:42,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741911_1089 (size=5523) 2024-12-03T09:25:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741911_1089 (size=5523) 2024-12-03T09:25:42,322 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741911_1089 (size=5523) 2024-12-03T09:25:42,322 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/a/b19ca9a6c207443d8cb27ddc6c846b42 2024-12-03T09:25:42,339 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/b/993ebe519c7f44029d150ae47ad04100 is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733217942218/Put/seqid=0 2024-12-03T09:25:42,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741912_1090 (size=5524) 2024-12-03T09:25:42,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741912_1090 (size=5524) 2024-12-03T09:25:42,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741912_1090 (size=5524) 2024-12-03T09:25:42,346 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/b/993ebe519c7f44029d150ae47ad04100 2024-12-03T09:25:42,363 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/c/1a3912b5942c40a1b32c94e28393b172 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733217942220/Put/seqid=0 2024-12-03T09:25:42,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741913_1091 (size=5457) 2024-12-03T09:25:42,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741913_1091 (size=5457) 2024-12-03T09:25:42,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741913_1091 (size=5457) 2024-12-03T09:25:42,370 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/c/1a3912b5942c40a1b32c94e28393b172 2024-12-03T09:25:42,376 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/a/b19ca9a6c207443d8cb27ddc6c846b42 as hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/a/b19ca9a6c207443d8cb27ddc6c846b42 2024-12-03T09:25:42,382 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/a/b19ca9a6c207443d8cb27ddc6c846b42, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T09:25:42,383 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/b/993ebe519c7f44029d150ae47ad04100 as hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/b/993ebe519c7f44029d150ae47ad04100 2024-12-03T09:25:42,388 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/b/993ebe519c7f44029d150ae47ad04100, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T09:25:42,389 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/.tmp/c/1a3912b5942c40a1b32c94e28393b172 as hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/c/1a3912b5942c40a1b32c94e28393b172 2024-12-03T09:25:42,394 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/c/1a3912b5942c40a1b32c94e28393b172, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T09:25:42,395 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 40fdae4b1a62d6d08dd8d623d21eb5c9 in 94ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T09:25:42,396 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/0000000000000000026 2024-12-03T09:25:42,397 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,397 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,398 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:42,399 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,402 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsAfterAbortingFlush/40fdae4b1a62d6d08dd8d623d21eb5c9/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T09:25:42,403 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 40fdae4b1a62d6d08dd8d623d21eb5c9; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59456339, jitterRate=-0.11403150856494904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:42,403 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 40fdae4b1a62d6d08dd8d623d21eb5c9 2024-12-03T09:25:42,404 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 40fdae4b1a62d6d08dd8d623d21eb5c9: Running coprocessor pre-open hook at 1733217942287Writing region info on filesystem at 1733217942287Initializing all the Stores at 1733217942288 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217942288Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217942288Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217942288Obtaining lock to block concurrent updates at 1733217942301 (+13 ms)Preparing flush snapshotting stores in 40fdae4b1a62d6d08dd8d623d21eb5c9 at 1733217942301Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733217942301Flushing stores of testReplayEditsAfterAbortingFlush,,1733217933056.40fdae4b1a62d6d08dd8d623d21eb5c9. at 1733217942301Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/a: creating writer at 1733217942301Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/a: appending metadata at 1733217942314 (+13 ms)Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/a: closing flushed file at 1733217942314Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/b: creating writer at 1733217942327 (+13 ms)Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/b: appending metadata at 1733217942339 (+12 ms)Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/b: closing flushed file at 1733217942339Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/c: creating writer at 1733217942350 (+11 ms)Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/c: appending metadata at 1733217942363 (+13 ms)Flushing 40fdae4b1a62d6d08dd8d623d21eb5c9/c: closing flushed file at 1733217942363Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ff071d8: reopening flushed file at 1733217942375 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@169fdb9c: reopening flushed file at 1733217942382 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1e3cc43e: reopening flushed file at 1733217942388 (+6 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 40fdae4b1a62d6d08dd8d623d21eb5c9 in 94ms, sequenceid=26, compaction requested=false; wal=null at 1733217942395 (+7 ms)Cleaning up temporary data from old regions at 1733217942397 (+2 ms)Running coprocessor post-open hooks at 1733217942403 (+6 ms)Region opened successfully at 1733217942404 (+1 ms) 2024-12-03T09:25:42,426 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsAfterAbortingFlush Thread=421 (was 422), OpenFileDescriptor=1182 (was 1126) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=190 (was 207), ProcessCount=11 (was 11), AvailableMemoryMB=13108 (was 13126) 2024-12-03T09:25:42,426 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T09:25:42,440 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=421, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=13107 2024-12-03T09:25:42,440 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T09:25:42,452 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:42,455 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:42,455 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:42,458 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-44803282, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-44803282, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:42,471 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-44803282/hregion-44803282.1733217942459, exclude list is [], retry=0 2024-12-03T09:25:42,474 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:42,474 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:42,474 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:42,477 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-44803282/hregion-44803282.1733217942459 2024-12-03T09:25:42,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:42,477 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 52df5865b70ff66bfe429cf8aa30d8e1, NAME => 'testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741915_1093 (size=61) 2024-12-03T09:25:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741915_1093 (size=61) 2024-12-03T09:25:42,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741915_1093 (size=61) 2024-12-03T09:25:42,493 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:42,494 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,495 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52df5865b70ff66bfe429cf8aa30d8e1 columnFamilyName a 2024-12-03T09:25:42,495 DEBUG [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:42,496 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(327): Store=52df5865b70ff66bfe429cf8aa30d8e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:42,496 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,496 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,497 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,497 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,497 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,499 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,501 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:42,501 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 52df5865b70ff66bfe429cf8aa30d8e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71922729, jitterRate=0.07173217833042145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 52df5865b70ff66bfe429cf8aa30d8e1: Writing region info on filesystem at 1733217942493Initializing all the Stores at 1733217942494 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217942494Cleaning up temporary data from old regions at 1733217942497 (+3 ms)Region opened successfully at 1733217942502 (+5 ms) 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 52df5865b70ff66bfe429cf8aa30d8e1, disabling compactions & flushes 2024-12-03T09:25:42,502 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. after waiting 0 ms 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,502 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,502 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 52df5865b70ff66bfe429cf8aa30d8e1: Waiting for close lock at 1733217942502Disabling compacts and flushes for region at 1733217942502Disabling writes for close at 1733217942502Writing region close event to WAL at 1733217942502Closed at 1733217942502 2024-12-03T09:25:42,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741914_1092 (size=95) 2024-12-03T09:25:42,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741914_1092 (size=95) 2024-12-03T09:25:42,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741914_1092 (size=95) 2024-12-03T09:25:42,507 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:42,507 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-44803282:(num 1733217942459) 2024-12-03T09:25:42,507 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:42,509 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:42,527 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, exclude list is [], retry=0 2024-12-03T09:25:42,530 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:42,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:42,531 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:42,533 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 2024-12-03T09:25:42,533 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:42,533 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 52df5865b70ff66bfe429cf8aa30d8e1, NAME => 'testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:42,534 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:42,534 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,534 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,536 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,537 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52df5865b70ff66bfe429cf8aa30d8e1 columnFamilyName a 2024-12-03T09:25:42,537 DEBUG [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:42,538 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(327): Store=52df5865b70ff66bfe429cf8aa30d8e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:42,538 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,539 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,540 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,541 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,541 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,542 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:42,543 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 52df5865b70ff66bfe429cf8aa30d8e1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65578804, jitterRate=-0.022799670696258545}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:42,544 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 52df5865b70ff66bfe429cf8aa30d8e1: Writing region info on filesystem at 1733217942534Initializing all the Stores at 1733217942535 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217942535Cleaning up temporary data from old regions at 1733217942541 (+6 ms)Region opened successfully at 1733217942544 (+3 ms) 2024-12-03T09:25:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 52df5865b70ff66bfe429cf8aa30d8e1, disabling compactions & flushes 2024-12-03T09:25:42,553 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. after waiting 0 ms 2024-12-03T09:25:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,553 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,553 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. 2024-12-03T09:25:42,553 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 52df5865b70ff66bfe429cf8aa30d8e1: Waiting for close lock at 1733217942553Disabling compacts and flushes for region at 1733217942553Disabling writes for close at 1733217942553Writing region close event to WAL at 1733217942553Closed at 1733217942553 2024-12-03T09:25:42,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741916_1094 (size=1050) 2024-12-03T09:25:42,556 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 not finished, retry = 0 2024-12-03T09:25:42,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741916_1094 (size=1050) 2024-12-03T09:25:42,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741916_1094 (size=1050) 2024-12-03T09:25:42,562 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T09:25:42,670 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, size=1.0 K (1050bytes) 2024-12-03T09:25:42,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 2024-12-03T09:25:42,670 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 after 0ms 2024-12-03T09:25:42,672 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:42,672 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 took 2ms 2024-12-03T09:25:42,674 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 so closing down 2024-12-03T09:25:42,675 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:42,676 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733217942510.temp 2024-12-03T09:25:42,677 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp 2024-12-03T09:25:42,677 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741917_1095 (size=1050) 2024-12-03T09:25:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741917_1095 (size=1050) 2024-12-03T09:25:42,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741917_1095 (size=1050) 2024-12-03T09:25:42,683 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:42,684 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp to hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 2024-12-03T09:25:42,684 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, size=1.0 K, length=1050, corrupted=false, cancelled=false 2024-12-03T09:25:42,684 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, journal: Splitting hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, size=1.0 K (1050bytes) at 1733217942670Finishing writing output for hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 so closing down at 1733217942675 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp at 1733217942677 (+2 ms)3 split writer threads finished at 1733217942677Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733217942683 (+6 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000003-wal.1733217942510.temp to hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 at 1733217942684 (+1 ms)Processed 10 edits across 1 Regions in 11 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510, size=1.0 K, length=1050, corrupted=false, cancelled=false at 1733217942684 2024-12-03T09:25:42,685 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217942510 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217942510 2024-12-03T09:25:42,686 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 2024-12-03T09:25:42,689 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:43,033 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:43,036 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:43,046 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217943036, exclude list is [], retry=0 2024-12-03T09:25:43,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:43,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:43,049 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:43,051 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217943036 2024-12-03T09:25:43,051 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:43,051 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 52df5865b70ff66bfe429cf8aa30d8e1, NAME => 'testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:43,051 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:43,051 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,051 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,054 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,056 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52df5865b70ff66bfe429cf8aa30d8e1 columnFamilyName a 2024-12-03T09:25:43,056 DEBUG [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,057 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(327): Store=52df5865b70ff66bfe429cf8aa30d8e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,057 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,057 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,059 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,059 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 2024-12-03T09:25:43,061 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:43,063 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 2024-12-03T09:25:43,063 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 52df5865b70ff66bfe429cf8aa30d8e1 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T09:25:43,083 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/.tmp/a/e834f283fb1140f38ec3aa90060ec7a5 is 79, key is testDatalossWhenInputError/a:x0/1733217942544/Put/seqid=0 2024-12-03T09:25:43,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741919_1097 (size=5808) 2024-12-03T09:25:43,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741919_1097 (size=5808) 2024-12-03T09:25:43,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741919_1097 (size=5808) 2024-12-03T09:25:43,089 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/.tmp/a/e834f283fb1140f38ec3aa90060ec7a5 2024-12-03T09:25:43,097 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/.tmp/a/e834f283fb1140f38ec3aa90060ec7a5 as hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/a/e834f283fb1140f38ec3aa90060ec7a5 2024-12-03T09:25:43,105 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/a/e834f283fb1140f38ec3aa90060ec7a5, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T09:25:43,105 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 52df5865b70ff66bfe429cf8aa30d8e1 in 42ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T09:25:43,106 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/0000000000000000012 2024-12-03T09:25:43,107 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,107 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,109 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,111 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T09:25:43,112 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 52df5865b70ff66bfe429cf8aa30d8e1; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64669323, jitterRate=-0.036351993680000305}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:43,112 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 52df5865b70ff66bfe429cf8aa30d8e1: Writing region info on filesystem at 1733217943051Initializing all the Stores at 1733217943054 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943054Obtaining lock to block concurrent updates at 1733217943063 (+9 ms)Preparing flush snapshotting stores in 52df5865b70ff66bfe429cf8aa30d8e1 at 1733217943063Finished memstore snapshotting testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733217943063Flushing stores of testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1. at 1733217943063Flushing 52df5865b70ff66bfe429cf8aa30d8e1/a: creating writer at 1733217943063Flushing 52df5865b70ff66bfe429cf8aa30d8e1/a: appending metadata at 1733217943082 (+19 ms)Flushing 52df5865b70ff66bfe429cf8aa30d8e1/a: closing flushed file at 1733217943082Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1985f55c: reopening flushed file at 1733217943095 (+13 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 52df5865b70ff66bfe429cf8aa30d8e1 in 42ms, sequenceid=12, compaction requested=false; wal=null at 1733217943105 (+10 ms)Cleaning up temporary data from old regions at 1733217943107 (+2 ms)Region opened successfully at 1733217943112 (+5 ms) 2024-12-03T09:25:43,115 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 52df5865b70ff66bfe429cf8aa30d8e1, NAME => 'testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:43,115 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733217942453.52df5865b70ff66bfe429cf8aa30d8e1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:43,115 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,115 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,116 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,117 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52df5865b70ff66bfe429cf8aa30d8e1 columnFamilyName a 2024-12-03T09:25:43,117 DEBUG [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,122 DEBUG [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/a/e834f283fb1140f38ec3aa90060ec7a5 2024-12-03T09:25:43,122 INFO [StoreOpener-52df5865b70ff66bfe429cf8aa30d8e1-1 {}] regionserver.HStore(327): Store=52df5865b70ff66bfe429cf8aa30d8e1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,122 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,124 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,124 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,124 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,126 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 52df5865b70ff66bfe429cf8aa30d8e1 2024-12-03T09:25:43,128 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testDatalossWhenInputError/52df5865b70ff66bfe429cf8aa30d8e1/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T09:25:43,129 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 52df5865b70ff66bfe429cf8aa30d8e1; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74918031, jitterRate=0.11636565625667572}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T09:25:43,129 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 52df5865b70ff66bfe429cf8aa30d8e1: Writing region info on filesystem at 1733217943115Initializing all the Stores at 1733217943116 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943116Cleaning up temporary data from old regions at 1733217943124 (+8 ms)Region opened successfully at 1733217943129 (+5 ms) 2024-12-03T09:25:43,146 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testDatalossWhenInputError Thread=432 (was 421) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:45968 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:35598 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:34108 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:35548 [Waiting for operation #14] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46036 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:34210 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=190 (was 190), ProcessCount=11 (was 11), AvailableMemoryMB=13102 (was 13107) 2024-12-03T09:25:43,146 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T09:25:43,159 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=432, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=190, ProcessCount=11, AvailableMemoryMB=13102 2024-12-03T09:25:43,159 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T09:25:43,175 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:43,176 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:43,177 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:43,179 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-15280203, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-15280203, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:43,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-15280203/hregion-15280203.1733217943179, exclude list is [], retry=0 2024-12-03T09:25:43,192 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:43,192 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:43,193 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:43,195 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-15280203/hregion-15280203.1733217943179 2024-12-03T09:25:43,196 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:43,196 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 08105e7573b8d1cd57a9ac8840837521, NAME => 'testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:43,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741921_1099 (size=63) 2024-12-03T09:25:43,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741921_1099 (size=63) 2024-12-03T09:25:43,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741921_1099 (size=63) 2024-12-03T09:25:43,207 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:43,209 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,210 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName a 2024-12-03T09:25:43,210 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,211 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,211 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,212 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName b 2024-12-03T09:25:43,212 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,213 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,213 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,214 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName c 2024-12-03T09:25:43,214 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,215 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,215 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,216 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,216 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,218 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,218 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,218 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:43,220 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,274 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:43,275 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08105e7573b8d1cd57a9ac8840837521; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58805765, jitterRate=-0.12372581660747528}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:43,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08105e7573b8d1cd57a9ac8840837521: Writing region info on filesystem at 1733217943208Initializing all the Stores at 1733217943208Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943208Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943208Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943209 (+1 ms)Cleaning up temporary data from old regions at 1733217943218 (+9 ms)Region opened successfully at 1733217943275 (+57 ms) 2024-12-03T09:25:43,275 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 08105e7573b8d1cd57a9ac8840837521, disabling compactions & flushes 2024-12-03T09:25:43,275 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:43,275 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:43,275 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. after waiting 0 ms 2024-12-03T09:25:43,275 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:43,276 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:43,276 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 08105e7573b8d1cd57a9ac8840837521: Waiting for close lock at 1733217943275Disabling compacts and flushes for region at 1733217943275Disabling writes for close at 1733217943275Writing region close event to WAL at 1733217943276 (+1 ms)Closed at 1733217943276 2024-12-03T09:25:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741920_1098 (size=95) 2024-12-03T09:25:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741920_1098 (size=95) 2024-12-03T09:25:43,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741920_1098 (size=95) 2024-12-03T09:25:43,281 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:43,281 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-15280203:(num 1733217943179) 2024-12-03T09:25:43,281 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:43,283 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:43,301 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, exclude list is [], retry=0 2024-12-03T09:25:43,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:43,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:43,304 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:43,306 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 2024-12-03T09:25:43,306 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:43,306 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 08105e7573b8d1cd57a9ac8840837521, NAME => 'testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:43,307 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:43,307 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,307 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,308 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,309 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName a 2024-12-03T09:25:43,309 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,310 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,310 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,310 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName b 2024-12-03T09:25:43,311 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,311 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,311 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,312 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName c 2024-12-03T09:25:43,312 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:43,312 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:43,312 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,313 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,314 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,315 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,315 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,316 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:43,317 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:43,318 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08105e7573b8d1cd57a9ac8840837521; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73773054, jitterRate=0.09930416941642761}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:43,318 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08105e7573b8d1cd57a9ac8840837521: Writing region info on filesystem at 1733217943307Initializing all the Stores at 1733217943308 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943308Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943308Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217943308Cleaning up temporary data from old regions at 1733217943315 (+7 ms)Region opened successfully at 1733217943318 (+3 ms) 2024-12-03T09:25:43,322 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733217943322/Put/seqid=0 2024-12-03T09:25:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741923_1101 (size=4875) 2024-12-03T09:25:43,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741923_1101 (size=4875) 2024-12-03T09:25:43,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741923_1101 (size=4875) 2024-12-03T09:25:43,334 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733217943334/Put/seqid=0 2024-12-03T09:25:43,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741924_1102 (size=4875) 2024-12-03T09:25:43,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741924_1102 (size=4875) 2024-12-03T09:25:43,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741924_1102 (size=4875) 2024-12-03T09:25:43,345 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733217943345/Put/seqid=0 2024-12-03T09:25:43,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741925_1103 (size=4875) 2024-12-03T09:25:43,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741925_1103 (size=4875) 2024-12-03T09:25:43,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741925_1103 (size=4875) 2024-12-03T09:25:43,352 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,356 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T09:25:43,356 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T09:25:43,356 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,359 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T09:25:43,359 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T09:25:43,359 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,362 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T09:25:43,362 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T09:25:43,362 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 08105e7573b8d1cd57a9ac8840837521 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T09:25:43,380 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp/a/97f7eedfcd064f0fa29825280d7ea366 is 55, key is testCompactedBulkLoadedFiles/a:a/1733217943318/Put/seqid=0 2024-12-03T09:25:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741926_1104 (size=5107) 2024-12-03T09:25:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741926_1104 (size=5107) 2024-12-03T09:25:43,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741926_1104 (size=5107) 2024-12-03T09:25:43,386 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp/a/97f7eedfcd064f0fa29825280d7ea366 2024-12-03T09:25:43,391 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp/a/97f7eedfcd064f0fa29825280d7ea366 as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366 2024-12-03T09:25:43,396 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T09:25:43,397 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 08105e7573b8d1cd57a9ac8840837521 in 35ms, sequenceid=4, compaction requested=false 2024-12-03T09:25:43,397 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 08105e7573b8d1cd57a9ac8840837521: 2024-12-03T09:25:43,398 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ 2024-12-03T09:25:43,399 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ 2024-12-03T09:25:43,400 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ 2024-12-03T09:25:43,401 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile0 into 08105e7573b8d1cd57a9ac8840837521/a as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ - updating store file list. 2024-12-03T09:25:43,405 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:43,405 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ into 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,405 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile0 into 08105e7573b8d1cd57a9ac8840837521/a (new location: hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_) 2024-12-03T09:25:43,406 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile1 into 08105e7573b8d1cd57a9ac8840837521/a as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ - updating store file list. 2024-12-03T09:25:43,410 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4c55798b596f4bedaf457438b48ceb9e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:43,410 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ into 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,410 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile1 into 08105e7573b8d1cd57a9ac8840837521/a (new location: hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_) 2024-12-03T09:25:43,411 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile2 into 08105e7573b8d1cd57a9ac8840837521/a as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ - updating store file list. 2024-12-03T09:25:43,415 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1d21d52760214826a02e47f28adc39ea_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:43,415 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ into 08105e7573b8d1cd57a9ac8840837521/a 2024-12-03T09:25:43,415 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:37427/hbase/testCompactedBulkLoadedFiles/hfile2 into 08105e7573b8d1cd57a9ac8840837521/a (new location: hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_) 2024-12-03T09:25:43,421 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T09:25:43,421 DEBUG [Time-limited test {}] regionserver.HStore(1541): 08105e7573b8d1cd57a9ac8840837521/a is initiating major compaction (all files) 2024-12-03T09:25:43,421 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 08105e7573b8d1cd57a9ac8840837521/a in testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:43,421 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_] into tmpdir=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp, totalSize=19.3 K 2024-12-03T09:25:43,422 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 97f7eedfcd064f0fa29825280d7ea366, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733217943318 2024-12-03T09:25:43,422 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T09:25:43,423 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4c55798b596f4bedaf457438b48ceb9e_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T09:25:43,423 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 1d21d52760214826a02e47f28adc39ea_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T09:25:43,434 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp/a/ae3e91e8f1294577a2e73d339671a84d is 55, key is testCompactedBulkLoadedFiles/a:a/1733217943318/Put/seqid=0 2024-12-03T09:25:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741927_1105 (size=6154) 2024-12-03T09:25:43,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741927_1105 (size=6154) 2024-12-03T09:25:43,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741927_1105 (size=6154) 2024-12-03T09:25:43,447 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/.tmp/a/ae3e91e8f1294577a2e73d339671a84d as hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/ae3e91e8f1294577a2e73d339671a84d 2024-12-03T09:25:43,452 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 08105e7573b8d1cd57a9ac8840837521/a of 08105e7573b8d1cd57a9ac8840837521 into ae3e91e8f1294577a2e73d339671a84d(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T09:25:43,452 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 08105e7573b8d1cd57a9ac8840837521: 2024-12-03T09:25:43,452 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T09:25:43,452 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T09:25:43,479 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, size=0 (0bytes) 2024-12-03T09:25:43,479 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 might be still open, length is 0 2024-12-03T09:25:43,479 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 2024-12-03T09:25:43,480 WARN [IPC Server handler 2 on default port 37427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-12-03T09:25:43,480 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 after 1ms 2024-12-03T09:25:43,675 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T09:25:43,675 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:43,676 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T09:25:43,676 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T09:25:45,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:34264 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:41153:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34264 dst: /127.0.0.1:41153 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41153 remote=/127.0.0.1:34264]. Total timeout mills is 60000, 57990 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:45,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:35630 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:34311:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35630 dst: /127.0.0.1:34311 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:45,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46048 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:36933:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46048 dst: /127.0.0.1:36933 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:45,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741922_1106 (size=1171) 2024-12-03T09:25:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741922_1106 (size=1171) 2024-12-03T09:25:45,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741922_1106 (size=1171) 2024-12-03T09:25:47,482 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 after 4003ms 2024-12-03T09:25:47,490 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:47,490 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 took 4011ms 2024-12-03T09:25:47,493 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284; continuing. 2024-12-03T09:25:47,493 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 so closing down 2024-12-03T09:25:47,493 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:47,494 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733217943284.temp 2024-12-03T09:25:47,496 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp 2024-12-03T09:25:47,496 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:47,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741928_1107 (size=546) 2024-12-03T09:25:47,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741928_1107 (size=546) 2024-12-03T09:25:47,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741928_1107 (size=546) 2024-12-03T09:25:47,503 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:47,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp to hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 2024-12-03T09:25:47,504 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T09:25:47,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, journal: Splitting hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, size=0 (0bytes) at 1733217943479Finishing writing output for hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 so closing down at 1733217947493 (+4014 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp at 1733217947496 (+3 ms)3 split writer threads finished at 1733217947496Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733217947503 (+7 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000003-wal.1733217943284.temp to hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 at 1733217947504 (+1 ms)Processed 5 edits across 1 Regions in 13 ms; skipped=3; WAL=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284, size=0, length=0, corrupted=false, cancelled=false at 1733217947504 2024-12-03T09:25:47,505 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217943284 2024-12-03T09:25:47,506 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 2024-12-03T09:25:47,506 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:47,508 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:47,518 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217947508, exclude list is [], retry=0 2024-12-03T09:25:47,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:47,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:47,521 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:47,523 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217947508 2024-12-03T09:25:47,523 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:47,523 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 08105e7573b8d1cd57a9ac8840837521, NAME => 'testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:47,523 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:47,524 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,524 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,525 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,526 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName a 2024-12-03T09:25:47,526 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,532 DEBUG [StoreFileOpener-08105e7573b8d1cd57a9ac8840837521-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 1d21d52760214826a02e47f28adc39ea_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:47,532 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ 2024-12-03T09:25:47,535 DEBUG [StoreFileOpener-08105e7573b8d1cd57a9ac8840837521-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:47,535 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ 2024-12-03T09:25:47,539 DEBUG [StoreFileOpener-08105e7573b8d1cd57a9ac8840837521-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 4c55798b596f4bedaf457438b48ceb9e_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T09:25:47,539 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ 2024-12-03T09:25:47,542 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366 2024-12-03T09:25:47,545 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/ae3e91e8f1294577a2e73d339671a84d 2024-12-03T09:25:47,546 WARN [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-12-03T09:25:47,546 WARN [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-12-03T09:25:47,546 WARN [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-12-03T09:25:47,546 WARN [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@76ebd70 2024-12-03T09:25:47,546 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366] to archive 2024-12-03T09:25:47,547 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T09:25:47,548 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ to hdfs://localhost:37427/hbase/archive/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/1d21d52760214826a02e47f28adc39ea_SeqId_4_ 2024-12-03T09:25:47,550 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ to hdfs://localhost:37427/hbase/archive/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_ 2024-12-03T09:25:47,551 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ to hdfs://localhost:37427/hbase/archive/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/4c55798b596f4bedaf457438b48ceb9e_SeqId_4_ 2024-12-03T09:25:47,552 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366 to hdfs://localhost:37427/hbase/archive/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/a/97f7eedfcd064f0fa29825280d7ea366 2024-12-03T09:25:47,552 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,552 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,553 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName b 2024-12-03T09:25:47,553 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,553 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,554 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,554 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 08105e7573b8d1cd57a9ac8840837521 columnFamilyName c 2024-12-03T09:25:47,554 DEBUG [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,555 INFO [StoreOpener-08105e7573b8d1cd57a9ac8840837521-1 {}] regionserver.HStore(327): Store=08105e7573b8d1cd57a9ac8840837521/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,555 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,555 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,557 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,557 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 2024-12-03T09:25:47,559 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:47,560 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 08105e7573b8d1cd57a9ac8840837521 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "08105e7573b8d1cd57a9ac8840837521" family_name: "a" compaction_input: "97f7eedfcd064f0fa29825280d7ea366" compaction_input: "2a11d7461bb94ca38f39b2dc14e9ae4c_SeqId_4_" compaction_input: "4c55798b596f4bedaf457438b48ceb9e_SeqId_4_" compaction_input: "1d21d52760214826a02e47f28adc39ea_SeqId_4_" compaction_output: "ae3e91e8f1294577a2e73d339671a84d" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T09:25:47,560 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T09:25:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 2024-12-03T09:25:47,561 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/0000000000000000008 2024-12-03T09:25:47,563 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,563 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,563 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:47,565 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 08105e7573b8d1cd57a9ac8840837521 2024-12-03T09:25:47,567 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testCompactedBulkLoadedFiles/08105e7573b8d1cd57a9ac8840837521/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T09:25:47,568 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 08105e7573b8d1cd57a9ac8840837521; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65867908, jitterRate=-0.018491685390472412}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:47,568 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 08105e7573b8d1cd57a9ac8840837521: Writing region info on filesystem at 1733217947524Initializing all the Stores at 1733217947525 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947525Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947525Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947525Cleaning up temporary data from old regions at 1733217947563 (+38 ms)Region opened successfully at 1733217947568 (+5 ms) 2024-12-03T09:25:47,570 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 08105e7573b8d1cd57a9ac8840837521, disabling compactions & flushes 2024-12-03T09:25:47,570 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:47,570 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:47,570 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. after waiting 0 ms 2024-12-03T09:25:47,570 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:47,571 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733217943175.08105e7573b8d1cd57a9ac8840837521. 2024-12-03T09:25:47,571 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 08105e7573b8d1cd57a9ac8840837521: Waiting for close lock at 1733217947570Disabling compacts and flushes for region at 1733217947570Disabling writes for close at 1733217947570Writing region close event to WAL at 1733217947571 (+1 ms)Closed at 1733217947571 2024-12-03T09:25:47,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741929_1108 (size=95) 2024-12-03T09:25:47,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741929_1108 (size=95) 2024-12-03T09:25:47,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741929_1108 (size=95) 2024-12-03T09:25:47,575 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:47,575 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733217947508) 2024-12-03T09:25:47,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testCompactedBulkLoadedFiles Thread=444 (was 432) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1300210318_22 at /127.0.0.1:34308 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:37427 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1300210318_22 at /127.0.0.1:46138 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:37427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1346 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=175 (was 190), ProcessCount=11 (was 11), AvailableMemoryMB=13092 (was 13102) 2024-12-03T09:25:47,590 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T09:25:47,601 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=444, OpenFileDescriptor=1346, MaxFileDescriptor=1048576, SystemLoadAverage=175, ProcessCount=11, AvailableMemoryMB=13092 2024-12-03T09:25:47,601 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1346 is superior to 1024 2024-12-03T09:25:47,613 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:47,614 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T09:25:47,615 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T09:25:47,617 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-35142294, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/hregion-35142294, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:47,628 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-35142294/hregion-35142294.1733217947618, exclude list is [], retry=0 2024-12-03T09:25:47,630 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:47,630 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:47,630 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:47,632 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-35142294/hregion-35142294.1733217947618 2024-12-03T09:25:47,632 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:47,632 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => af38b908b2a775f163526daaa64a5989, NAME => 'testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:37427/hbase 2024-12-03T09:25:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741931_1110 (size=67) 2024-12-03T09:25:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741931_1110 (size=67) 2024-12-03T09:25:47,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741931_1110 (size=67) 2024-12-03T09:25:47,641 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:47,642 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,643 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName a 2024-12-03T09:25:47,643 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,644 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,644 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,645 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName b 2024-12-03T09:25:47,645 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,645 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,646 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,647 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName c 2024-12-03T09:25:47,647 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,647 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,647 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,648 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,649 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,649 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,649 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:47,650 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T09:25:47,653 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened af38b908b2a775f163526daaa64a5989; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70007383, jitterRate=0.043191298842430115}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for af38b908b2a775f163526daaa64a5989: Writing region info on filesystem at 1733217947641Initializing all the Stores at 1733217947642 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947642Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947642Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947642Cleaning up temporary data from old regions at 1733217947649 (+7 ms)Region opened successfully at 1733217947653 (+4 ms) 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing af38b908b2a775f163526daaa64a5989, disabling compactions & flushes 2024-12-03T09:25:47,653 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. after waiting 0 ms 2024-12-03T09:25:47,653 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,654 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,654 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for af38b908b2a775f163526daaa64a5989: Waiting for close lock at 1733217947653Disabling compacts and flushes for region at 1733217947653Disabling writes for close at 1733217947653Writing region close event to WAL at 1733217947654 (+1 ms)Closed at 1733217947654 2024-12-03T09:25:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741930_1109 (size=95) 2024-12-03T09:25:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741930_1109 (size=95) 2024-12-03T09:25:47,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741930_1109 (size=95) 2024-12-03T09:25:47,658 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:47,658 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-35142294:(num 1733217947618) 2024-12-03T09:25:47,658 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:47,660 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:47,673 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, exclude list is [], retry=0 2024-12-03T09:25:47,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:47,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:47,676 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:47,678 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 2024-12-03T09:25:47,678 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521)] 2024-12-03T09:25:47,678 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => af38b908b2a775f163526daaa64a5989, NAME => 'testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:47,678 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:47,678 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,678 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,680 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,680 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName a 2024-12-03T09:25:47,680 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,681 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,681 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,682 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName b 2024-12-03T09:25:47,682 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,682 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,682 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,683 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName c 2024-12-03T09:25:47,683 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,683 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,684 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,684 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,685 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,686 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,686 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,687 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:47,688 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,688 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened af38b908b2a775f163526daaa64a5989; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64422800, jitterRate=-0.04002547264099121}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:47,689 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for af38b908b2a775f163526daaa64a5989: Writing region info on filesystem at 1733217947678Initializing all the Stores at 1733217947679 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947679Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947679Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947680 (+1 ms)Cleaning up temporary data from old regions at 1733217947686 (+6 ms)Region opened successfully at 1733217947689 (+3 ms) 2024-12-03T09:25:47,695 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing af38b908b2a775f163526daaa64a5989 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T09:25:47,708 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/476fc6f91c854876986a2b64c2903319 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733217947689/Put/seqid=0 2024-12-03T09:25:47,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741933_1112 (size=5958) 2024-12-03T09:25:47,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741933_1112 (size=5958) 2024-12-03T09:25:47,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741933_1112 (size=5958) 2024-12-03T09:25:47,715 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/476fc6f91c854876986a2b64c2903319 2024-12-03T09:25:47,720 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/476fc6f91c854876986a2b64c2903319 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/476fc6f91c854876986a2b64c2903319 2024-12-03T09:25:47,725 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/476fc6f91c854876986a2b64c2903319, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T09:25:47,726 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for af38b908b2a775f163526daaa64a5989 in 31ms, sequenceid=13, compaction requested=false 2024-12-03T09:25:47,726 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for af38b908b2a775f163526daaa64a5989: 2024-12-03T09:25:47,740 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing af38b908b2a775f163526daaa64a5989, disabling compactions & flushes 2024-12-03T09:25:47,740 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,740 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,740 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. after waiting 0 ms 2024-12-03T09:25:47,740 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,741 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,741 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:47,741 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for af38b908b2a775f163526daaa64a5989: Waiting for close lock at 1733217947740Disabling compacts and flushes for region at 1733217947740Disabling writes for close at 1733217947740Writing region close event to WAL at 1733217947741 (+1 ms)Closed at 1733217947741 2024-12-03T09:25:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741932_1111 (size=3345) 2024-12-03T09:25:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741932_1111 (size=3345) 2024-12-03T09:25:47,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741932_1111 (size=3345) 2024-12-03T09:25:47,756 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, size=3.3 K (3345bytes) 2024-12-03T09:25:47,756 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 2024-12-03T09:25:47,757 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 after 1ms 2024-12-03T09:25:47,759 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:47,759 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 took 3ms 2024-12-03T09:25:47,761 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 so closing down 2024-12-03T09:25:47,761 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:47,762 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733217947660.temp 2024-12-03T09:25:47,763 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp 2024-12-03T09:25:47,764 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:47,764 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T09:25:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741934_1113 (size=2944) 2024-12-03T09:25:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741934_1113 (size=2944) 2024-12-03T09:25:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741934_1113 (size=2944) 2024-12-03T09:25:47,775 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:47,776 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 2024-12-03T09:25:47,776 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, size=3.3 K, length=3345, corrupted=false, cancelled=false 2024-12-03T09:25:47,776 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, journal: Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, size=3.3 K (3345bytes) at 1733217947756Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 so closing down at 1733217947761 (+5 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp at 1733217947763 (+2 ms)3 split writer threads finished at 1733217947764 (+1 ms)Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733217947775 (+11 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000003-wal.1733217947660.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 at 1733217947776 (+1 ms)Processed 32 edits across 1 Regions in 17 ms; skipped=2; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660, size=3.3 K, length=3345, corrupted=false, cancelled=false at 1733217947776 2024-12-03T09:25:47,778 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947660 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217947660 2024-12-03T09:25:47,778 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 2024-12-03T09:25:47,779 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:47,780 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:47,792 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, exclude list is [], retry=0 2024-12-03T09:25:47,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:47,794 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:47,795 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:47,796 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 2024-12-03T09:25:47,796 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40249:40249),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40327:40327)] 2024-12-03T09:25:47,796 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => af38b908b2a775f163526daaa64a5989, NAME => 'testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.', STARTKEY => '', ENDKEY => ''} 2024-12-03T09:25:47,797 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:47,797 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,797 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,799 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,800 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName a 2024-12-03T09:25:47,800 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,806 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/476fc6f91c854876986a2b64c2903319 2024-12-03T09:25:47,806 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,806 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,807 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName b 2024-12-03T09:25:47,807 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,808 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,808 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,808 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName c 2024-12-03T09:25:47,808 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:47,809 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:47,809 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,810 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,811 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,811 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 2024-12-03T09:25:47,813 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:47,815 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 2024-12-03T09:25:47,815 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing af38b908b2a775f163526daaa64a5989 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T09:25:47,827 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/c9da5a3027184b3aaebcc15fedc1a28b is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733217947726/Put/seqid=0 2024-12-03T09:25:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741936_1115 (size=5958) 2024-12-03T09:25:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741936_1115 (size=5958) 2024-12-03T09:25:47,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741936_1115 (size=5958) 2024-12-03T09:25:47,833 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/c9da5a3027184b3aaebcc15fedc1a28b 2024-12-03T09:25:47,851 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/16615030d563444ca79683cef0450cfd is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733217947732/Put/seqid=0 2024-12-03T09:25:47,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741937_1116 (size=5958) 2024-12-03T09:25:47,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741937_1116 (size=5958) 2024-12-03T09:25:47,856 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741937_1116 (size=5958) 2024-12-03T09:25:47,857 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/16615030d563444ca79683cef0450cfd 2024-12-03T09:25:47,862 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/c9da5a3027184b3aaebcc15fedc1a28b as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/b/c9da5a3027184b3aaebcc15fedc1a28b 2024-12-03T09:25:47,867 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/b/c9da5a3027184b3aaebcc15fedc1a28b, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T09:25:47,868 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/16615030d563444ca79683cef0450cfd as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/c/16615030d563444ca79683cef0450cfd 2024-12-03T09:25:47,874 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/c/16615030d563444ca79683cef0450cfd, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T09:25:47,874 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for af38b908b2a775f163526daaa64a5989 in 59ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T09:25:47,875 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000035 2024-12-03T09:25:47,876 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,876 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,877 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:47,878 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:47,880 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T09:25:47,881 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened af38b908b2a775f163526daaa64a5989; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73197946, jitterRate=0.09073439240455627}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:47,881 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for af38b908b2a775f163526daaa64a5989: Writing region info on filesystem at 1733217947797Initializing all the Stores at 1733217947797Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947798 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947799 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217947799Obtaining lock to block concurrent updates at 1733217947815 (+16 ms)Preparing flush snapshotting stores in af38b908b2a775f163526daaa64a5989 at 1733217947815Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733217947815Flushing stores of testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. at 1733217947815Flushing af38b908b2a775f163526daaa64a5989/b: creating writer at 1733217947815Flushing af38b908b2a775f163526daaa64a5989/b: appending metadata at 1733217947826 (+11 ms)Flushing af38b908b2a775f163526daaa64a5989/b: closing flushed file at 1733217947827 (+1 ms)Flushing af38b908b2a775f163526daaa64a5989/c: creating writer at 1733217947838 (+11 ms)Flushing af38b908b2a775f163526daaa64a5989/c: appending metadata at 1733217947850 (+12 ms)Flushing af38b908b2a775f163526daaa64a5989/c: closing flushed file at 1733217947850Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40a7bcea: reopening flushed file at 1733217947861 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5ba5609: reopening flushed file at 1733217947868 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for af38b908b2a775f163526daaa64a5989 in 59ms, sequenceid=35, compaction requested=false; wal=null at 1733217947874 (+6 ms)Cleaning up temporary data from old regions at 1733217947876 (+2 ms)Region opened successfully at 1733217947881 (+5 ms) 2024-12-03T09:25:47,941 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, size=0 (0bytes) 2024-12-03T09:25:47,942 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 might be still open, length is 0 2024-12-03T09:25:47,942 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 2024-12-03T09:25:47,942 WARN [IPC Server handler 4 on default port 37427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-12-03T09:25:47,942 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 after 0ms 2024-12-03T09:25:48,462 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:34364 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:41153:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:34364 dst: /127.0.0.1:41153 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:41153 remote=/127.0.0.1:34364]. Total timeout mills is 60000, 59444 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:48,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:35794 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:34311:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35794 dst: /127.0.0.1:34311 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:48,463 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1431312456_22 at /127.0.0.1:46176 [Receiving block BP-1349158104-172.17.0.2-1733217908790:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:36933:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46176 dst: /127.0.0.1:36933 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:48,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741935_1117 (size=2936) 2024-12-03T09:25:48,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741935_1117 (size=2936) 2024-12-03T09:25:48,467 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741935_1117 (size=2936) 2024-12-03T09:25:51,944 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 after 4001ms 2024-12-03T09:25:51,950 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780: isRecoveredEdits=false, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:51,950 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 took 4009ms 2024-12-03T09:25:51,953 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780; continuing. 2024-12-03T09:25:51,954 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 so closing down 2024-12-03T09:25:51,954 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T09:25:51,954 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733217947780.temp 2024-12-03T09:25:51,956 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp 2024-12-03T09:25:51,956 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T09:25:51,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741938_1118 (size=2944) 2024-12-03T09:25:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741938_1118 (size=2944) 2024-12-03T09:25:51,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741938_1118 (size=2944) 2024-12-03T09:25:51,962 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T09:25:51,963 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 2024-12-03T09:25:51,964 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T09:25:51,964 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, journal: Splitting hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, size=0 (0bytes) at 1733217947942Finishing writing output for hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 so closing down at 1733217951954 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp at 1733217951956 (+2 ms)3 split writer threads finished at 1733217951956Closed recovered edits writer path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733217951962 (+6 ms)Rename recovered edits hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000037-wal.1733217947780.temp to hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 at 1733217951963 (+1 ms)Processed 30 edits across 1 Regions in 13 ms; skipped=0; WAL=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780, size=0, length=0, corrupted=false, cancelled=false at 1733217951964 (+1 ms) 2024-12-03T09:25:51,965 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 to hdfs://localhost:37427/hbase/oldWALs/wal.1733217947780 2024-12-03T09:25:51,966 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 2024-12-03T09:25:51,966 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T09:25:51,968 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:37427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612, archiveDir=hdfs://localhost:37427/hbase/oldWALs, maxLogs=32 2024-12-03T09:25:51,978 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217951968, exclude list is [], retry=0 2024-12-03T09:25:51,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36933,DS-5878274c-1618-4fe8-9285-33433066f5e3,DISK] 2024-12-03T09:25:51,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34311,DS-92f817a3-291f-4319-bace-a448fa4c2594,DISK] 2024-12-03T09:25:51,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:41153,DS-6cdfbc66-232b-48a4-adbb-355e49d63215,DISK] 2024-12-03T09:25:51,982 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217951968 2024-12-03T09:25:51,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40327:40327),(127.0.0.1/127.0.0.1:37521:37521),(127.0.0.1/127.0.0.1:40249:40249)] 2024-12-03T09:25:51,983 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T09:25:51,984 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:51,985 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName a 2024-12-03T09:25:51,985 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:51,990 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/476fc6f91c854876986a2b64c2903319 2024-12-03T09:25:51,990 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:51,990 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:51,991 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName b 2024-12-03T09:25:51,991 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:51,996 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/b/c9da5a3027184b3aaebcc15fedc1a28b 2024-12-03T09:25:51,996 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:51,996 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:51,997 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region af38b908b2a775f163526daaa64a5989 columnFamilyName c 2024-12-03T09:25:51,997 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T09:25:52,001 DEBUG [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/c/16615030d563444ca79683cef0450cfd 2024-12-03T09:25:52,001 INFO [StoreOpener-af38b908b2a775f163526daaa64a5989-1 {}] regionserver.HStore(327): Store=af38b908b2a775f163526daaa64a5989/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T09:25:52,001 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,002 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,004 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,004 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 2024-12-03T09:25:52,006 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(321): Initializing compression context for hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066: isRecoveredEdits=true, hasTagCompression=true, hasValueCompression=true, valueCompressionType=GZ 2024-12-03T09:25:52,011 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 2024-12-03T09:25:52,011 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing af38b908b2a775f163526daaa64a5989 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T09:25:52,024 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/053e597c3a7d431dad044daa1b9af039 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733217947888/Put/seqid=0 2024-12-03T09:25:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741940_1120 (size=5958) 2024-12-03T09:25:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741940_1120 (size=5958) 2024-12-03T09:25:52,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741940_1120 (size=5958) 2024-12-03T09:25:52,030 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/053e597c3a7d431dad044daa1b9af039 2024-12-03T09:25:52,048 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/b48cf1792a4242b89c0a41f57822aaba is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733217947894/Put/seqid=0 2024-12-03T09:25:52,053 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741941_1121 (size=5958) 2024-12-03T09:25:52,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741941_1121 (size=5958) 2024-12-03T09:25:52,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741941_1121 (size=5958) 2024-12-03T09:25:52,054 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/b48cf1792a4242b89c0a41f57822aaba 2024-12-03T09:25:52,071 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/11b3a1b969a4436d8fe901112c80ad10 is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733217947900/Put/seqid=0 2024-12-03T09:25:52,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741942_1122 (size=5958) 2024-12-03T09:25:52,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741942_1122 (size=5958) 2024-12-03T09:25:52,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741942_1122 (size=5958) 2024-12-03T09:25:52,077 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/11b3a1b969a4436d8fe901112c80ad10 2024-12-03T09:25:52,082 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/a/053e597c3a7d431dad044daa1b9af039 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/053e597c3a7d431dad044daa1b9af039 2024-12-03T09:25:52,086 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/a/053e597c3a7d431dad044daa1b9af039, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T09:25:52,087 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/b/b48cf1792a4242b89c0a41f57822aaba as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/b/b48cf1792a4242b89c0a41f57822aaba 2024-12-03T09:25:52,091 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/b/b48cf1792a4242b89c0a41f57822aaba, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T09:25:52,092 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/.tmp/c/11b3a1b969a4436d8fe901112c80ad10 as hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/c/11b3a1b969a4436d8fe901112c80ad10 2024-12-03T09:25:52,096 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/c/11b3a1b969a4436d8fe901112c80ad10, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T09:25:52,096 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for af38b908b2a775f163526daaa64a5989 in 85ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T09:25:52,097 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/0000000000000000066 2024-12-03T09:25:52,098 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,098 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,098 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T09:25:52,099 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for af38b908b2a775f163526daaa64a5989 2024-12-03T09:25:52,101 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/hbase/data/default/testReplayEditsWrittenViaHRegion/af38b908b2a775f163526daaa64a5989/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T09:25:52,102 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened af38b908b2a775f163526daaa64a5989; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64216396, jitterRate=-0.04310113191604614}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T09:25:52,102 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for af38b908b2a775f163526daaa64a5989: Writing region info on filesystem at 1733217951983Initializing all the Stores at 1733217951984 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217951984Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217951984Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733217951984Obtaining lock to block concurrent updates at 1733217952011 (+27 ms)Preparing flush snapshotting stores in af38b908b2a775f163526daaa64a5989 at 1733217952011Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733217952011Flushing stores of testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. at 1733217952011Flushing af38b908b2a775f163526daaa64a5989/a: creating writer at 1733217952011Flushing af38b908b2a775f163526daaa64a5989/a: appending metadata at 1733217952024 (+13 ms)Flushing af38b908b2a775f163526daaa64a5989/a: closing flushed file at 1733217952024Flushing af38b908b2a775f163526daaa64a5989/b: creating writer at 1733217952034 (+10 ms)Flushing af38b908b2a775f163526daaa64a5989/b: appending metadata at 1733217952048 (+14 ms)Flushing af38b908b2a775f163526daaa64a5989/b: closing flushed file at 1733217952048Flushing af38b908b2a775f163526daaa64a5989/c: creating writer at 1733217952058 (+10 ms)Flushing af38b908b2a775f163526daaa64a5989/c: appending metadata at 1733217952071 (+13 ms)Flushing af38b908b2a775f163526daaa64a5989/c: closing flushed file at 1733217952071Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73fdc1a2: reopening flushed file at 1733217952081 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5622602b: reopening flushed file at 1733217952086 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7cc02cf9: reopening flushed file at 1733217952092 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for af38b908b2a775f163526daaa64a5989 in 85ms, sequenceid=66, compaction requested=false; wal=null at 1733217952096 (+4 ms)Cleaning up temporary data from old regions at 1733217952098 (+2 ms)Region opened successfully at 1733217952102 (+4 ms) 2024-12-03T09:25:52,114 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing af38b908b2a775f163526daaa64a5989, disabling compactions & flushes 2024-12-03T09:25:52,114 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:52,114 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:52,114 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. after waiting 0 ms 2024-12-03T09:25:52,114 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:52,118 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733217947613.af38b908b2a775f163526daaa64a5989. 2024-12-03T09:25:52,118 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for af38b908b2a775f163526daaa64a5989: Waiting for close lock at 1733217952114Disabling compacts and flushes for region at 1733217952114Disabling writes for close at 1733217952114Writing region close event to WAL at 1733217952117 (+3 ms)Closed at 1733217952118 (+1 ms) 2024-12-03T09:25:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741939_1119 (size=95) 2024-12-03T09:25:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741939_1119 (size=95) 2024-12-03T09:25:52,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741939_1119 (size=95) 2024-12-03T09:25:52,122 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T09:25:52,122 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733217951968) 2024-12-03T09:25:52,137 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplayValueCompression#testReplayEditsWrittenViaHRegion Thread=451 (was 444) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1513056918_22 at /127.0.0.1:58638 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:37427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1513056918_22 at /127.0.0.1:42790 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1513056918_22 at /127.0.0.1:36980 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (754041887) connection to localhost/127.0.0.1:37427 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1346) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=168 (was 175), ProcessCount=11 (was 11), AvailableMemoryMB=13063 (was 13092) 2024-12-03T09:25:52,138 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-12-03T09:25:52,138 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T09:25:52,138 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:25:52,138 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:25:52,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,139 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,139 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T09:25:52,139 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T09:25:52,139 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1962663776, stopped=false 2024-12-03T09:25:52,139 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=699c9baf3d29,33521,1733217913369 2024-12-03T09:25:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T09:25:52,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:52,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:52,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:52,204 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T09:25:52,204 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:52,204 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:52,204 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T09:25:52,205 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T09:25:52,205 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:25:52,205 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,205 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '699c9baf3d29,40727,1733217914150' ***** 2024-12-03T09:25:52,205 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T09:25:52,205 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '699c9baf3d29,46197,1733217914195' ***** 2024-12-03T09:25:52,205 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T09:25:52,205 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T09:25:52,205 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T09:25:52,205 INFO [RS:1;699c9baf3d29:40727 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T09:25:52,205 INFO [RS:2;699c9baf3d29:46197 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T09:25:52,205 INFO [RS:2;699c9baf3d29:46197 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T09:25:52,206 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(3091): Received CLOSE for 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(959): stopping server 699c9baf3d29,40727,1733217914150 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;699c9baf3d29:40727. 2024-12-03T09:25:52,206 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(959): stopping server 699c9baf3d29,46197,1733217914195 2024-12-03T09:25:52,206 INFO [RS:2;699c9baf3d29:46197 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:25:52,206 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T09:25:52,206 DEBUG [RS:1;699c9baf3d29:40727 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:25:52,206 DEBUG [RS:1;699c9baf3d29:40727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,206 INFO [RS:2;699c9baf3d29:46197 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;699c9baf3d29:46197. 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T09:25:52,206 DEBUG [RS:2;699c9baf3d29:46197 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T09:25:52,206 DEBUG [RS:2;699c9baf3d29:46197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T09:25:52,206 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T09:25:52,206 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1325): Online Regions={60a49919b1e567cff342967e23c2879b=testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b.} 2024-12-03T09:25:52,206 DEBUG [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1351): Waiting on 60a49919b1e567cff342967e23c2879b 2024-12-03T09:25:52,206 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T09:25:52,206 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 60a49919b1e567cff342967e23c2879b, disabling compactions & flushes 2024-12-03T09:25:52,206 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:52,206 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T09:25:52,206 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:52,207 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. after waiting 0 ms 2024-12-03T09:25:52,207 DEBUG [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T09:25:52,207 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T09:25:52,207 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T09:25:52,207 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-12-03T09:25:52,210 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/default/testReplayEditsAfterRegionMovedWithMultiCF/60a49919b1e567cff342967e23c2879b/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T09:25:52,211 INFO [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:52,211 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 60a49919b1e567cff342967e23c2879b: Waiting for close lock at 1733217952206Running coprocessor pre-close hooks at 1733217952206Disabling compacts and flushes for region at 1733217952206Disabling writes for close at 1733217952207 (+1 ms)Writing region close event to WAL at 1733217952207Running coprocessor post-close hooks at 1733217952211 (+4 ms)Closed at 1733217952211 2024-12-03T09:25:52,211 DEBUG [RS_CLOSE_REGION-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b. 2024-12-03T09:25:52,212 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/4b938a36de0941cfa86f7841c6d5eb10 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733217928199.60a49919b1e567cff342967e23c2879b./info:regioninfo/1733217932211/Put/seqid=0 2024-12-03T09:25:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741943_1123 (size=6778) 2024-12-03T09:25:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741943_1123 (size=6778) 2024-12-03T09:25:52,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741943_1123 (size=6778) 2024-12-03T09:25:52,220 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/4b938a36de0941cfa86f7841c6d5eb10 2024-12-03T09:25:52,225 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/.tmp/info/4b938a36de0941cfa86f7841c6d5eb10 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/info/4b938a36de0941cfa86f7841c6d5eb10 2024-12-03T09:25:52,230 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/info/4b938a36de0941cfa86f7841c6d5eb10, entries=8, sequenceid=23, filesize=6.6 K 2024-12-03T09:25:52,231 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false 2024-12-03T09:25:52,231 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T09:25:52,256 INFO [regionserver/699c9baf3d29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:52,256 INFO [regionserver/699c9baf3d29:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:52,258 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-12-03T09:25:52,258 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T09:25:52,259 INFO [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T09:25:52,259 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733217952207Running coprocessor pre-close hooks at 1733217952207Disabling compacts and flushes for region at 1733217952207Disabling writes for close at 1733217952207Obtaining lock to block concurrent updates at 1733217952207Preparing flush snapshotting stores in 1588230740 at 1733217952207Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1733217952207Flushing stores of hbase:meta,,1.1588230740 at 1733217952208 (+1 ms)Flushing 1588230740/info: creating writer at 1733217952208Flushing 1588230740/info: appending metadata at 1733217952211 (+3 ms)Flushing 1588230740/info: closing flushed file at 1733217952211Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40ff0333: reopening flushed file at 1733217952225 (+14 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 24ms, sequenceid=23, compaction requested=false at 1733217952231 (+6 ms)Writing region close event to WAL at 1733217952233 (+2 ms)Running coprocessor post-close hooks at 1733217952258 (+25 ms)Closed at 1733217952259 (+1 ms) 2024-12-03T09:25:52,259 DEBUG [RS_CLOSE_META-regionserver/699c9baf3d29:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T09:25:52,407 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(976): stopping server 699c9baf3d29,46197,1733217914195; all regions closed. 2024-12-03T09:25:52,407 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(976): stopping server 699c9baf3d29,40727,1733217914150; all regions closed. 2024-12-03T09:25:52,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741835_1011 (size=722) 2024-12-03T09:25:52,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741835_1011 (size=722) 2024-12-03T09:25:52,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741835_1011 (size=722) 2024-12-03T09:25:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741893_1071 (size=1673) 2024-12-03T09:25:52,414 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/WALs/699c9baf3d29,40727,1733217914150/699c9baf3d29%2C40727%2C1733217914150.meta.1733217931389.meta not finished, retry = 0 2024-12-03T09:25:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741893_1071 (size=1673) 2024-12-03T09:25:52,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741893_1071 (size=1673) 2024-12-03T09:25:52,416 DEBUG [RS:2;699c9baf3d29:46197 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs 2024-12-03T09:25:52,416 INFO [RS:2;699c9baf3d29:46197 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 699c9baf3d29%2C46197%2C1733217914195:(num 1733217915683) 2024-12-03T09:25:52,416 DEBUG [RS:2;699c9baf3d29:46197 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,416 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:52,416 INFO [RS:2;699c9baf3d29:46197 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] hbase.ChoreService(370): Chore service for: regionserver/699c9baf3d29:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T09:25:52,417 INFO [regionserver/699c9baf3d29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:25:52,417 INFO [RS:2;699c9baf3d29:46197 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46197 2024-12-03T09:25:52,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:25:52,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/699c9baf3d29,46197,1733217914195 2024-12-03T09:25:52,424 INFO [RS:2;699c9baf3d29:46197 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:25:52,435 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [699c9baf3d29,46197,1733217914195] 2024-12-03T09:25:52,445 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/699c9baf3d29,46197,1733217914195 already deleted, retry=false 2024-12-03T09:25:52,446 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 699c9baf3d29,46197,1733217914195 expired; onlineServers=1 2024-12-03T09:25:52,510 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-03T09:25:52,510 INFO [regionserver/699c9baf3d29:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-03T09:25:52,520 DEBUG [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs 2024-12-03T09:25:52,521 INFO [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 699c9baf3d29%2C40727%2C1733217914150.meta:.meta(num 1733217931389) 2024-12-03T09:25:52,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741833_1009 (size=1702) 2024-12-03T09:25:52,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741833_1009 (size=1702) 2024-12-03T09:25:52,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741833_1009 (size=1702) 2024-12-03T09:25:52,528 DEBUG [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/oldWALs 2024-12-03T09:25:52,528 INFO [RS:1;699c9baf3d29:40727 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 699c9baf3d29%2C40727%2C1733217914150:(num 1733217915683) 2024-12-03T09:25:52,528 DEBUG [RS:1;699c9baf3d29:40727 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T09:25:52,528 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T09:25:52,528 INFO [RS:1;699c9baf3d29:40727 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:25:52,528 INFO [RS:1;699c9baf3d29:40727 {}] hbase.ChoreService(370): Chore service for: regionserver/699c9baf3d29:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T09:25:52,529 INFO [RS:1;699c9baf3d29:40727 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:25:52,529 INFO [regionserver/699c9baf3d29:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:25:52,529 INFO [RS:1;699c9baf3d29:40727 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40727 2024-12-03T09:25:52,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,535 INFO [RS:2;699c9baf3d29:46197 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:25:52,535 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46197-0x1019a8282cb0003, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,535 INFO [RS:2;699c9baf3d29:46197 {}] regionserver.HRegionServer(1031): Exiting; stopping=699c9baf3d29,46197,1733217914195; zookeeper connection closed. 2024-12-03T09:25:52,536 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63c17a96 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63c17a96 2024-12-03T09:25:52,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T09:25:52,540 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/699c9baf3d29,40727,1733217914150 2024-12-03T09:25:52,540 INFO [RS:1;699c9baf3d29:40727 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:25:52,551 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [699c9baf3d29,40727,1733217914150] 2024-12-03T09:25:52,561 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/699c9baf3d29,40727,1733217914150 already deleted, retry=false 2024-12-03T09:25:52,561 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 699c9baf3d29,40727,1733217914150 expired; onlineServers=0 2024-12-03T09:25:52,561 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '699c9baf3d29,33521,1733217913369' ***** 2024-12-03T09:25:52,561 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T09:25:52,561 INFO [M:0;699c9baf3d29:33521 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T09:25:52,561 INFO [M:0;699c9baf3d29:33521 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T09:25:52,562 DEBUG [M:0;699c9baf3d29:33521 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T09:25:52,562 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T09:25:52,562 DEBUG [M:0;699c9baf3d29:33521 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T09:25:52,562 DEBUG [master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217915306 {}] cleaner.HFileCleaner(306): Exit Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.large.0-1733217915306,5,FailOnTimeoutGroup] 2024-12-03T09:25:52,562 DEBUG [master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217915312 {}] cleaner.HFileCleaner(306): Exit Thread[master/699c9baf3d29:0:becomeActiveMaster-HFileCleaner.small.0-1733217915312,5,FailOnTimeoutGroup] 2024-12-03T09:25:52,562 INFO [M:0;699c9baf3d29:33521 {}] hbase.ChoreService(370): Chore service for: master/699c9baf3d29:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T09:25:52,562 INFO [M:0;699c9baf3d29:33521 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T09:25:52,562 DEBUG [M:0;699c9baf3d29:33521 {}] master.HMaster(1795): Stopping service threads 2024-12-03T09:25:52,562 INFO [M:0;699c9baf3d29:33521 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T09:25:52,562 INFO [M:0;699c9baf3d29:33521 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T09:25:52,563 INFO [M:0;699c9baf3d29:33521 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T09:25:52,563 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T09:25:52,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T09:25:52,572 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T09:25:52,572 DEBUG [M:0;699c9baf3d29:33521 {}] zookeeper.ZKUtil(347): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T09:25:52,572 WARN [M:0;699c9baf3d29:33521 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T09:25:52,573 INFO [M:0;699c9baf3d29:33521 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/.lastflushedseqids 2024-12-03T09:25:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741944_1124 (size=119) 2024-12-03T09:25:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741944_1124 (size=119) 2024-12-03T09:25:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741944_1124 (size=119) 2024-12-03T09:25:52,584 INFO [M:0;699c9baf3d29:33521 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T09:25:52,585 INFO [M:0;699c9baf3d29:33521 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T09:25:52,585 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T09:25:52,585 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:52,585 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:52,585 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T09:25:52,585 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:52,585 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.41 KB heapSize=102.71 KB 2024-12-03T09:25:52,603 DEBUG [M:0;699c9baf3d29:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef820256c4f94fb5b7c767f2186b302d is 82, key is hbase:meta,,1/info:regioninfo/1733217931539/Put/seqid=0 2024-12-03T09:25:52,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741945_1125 (size=6063) 2024-12-03T09:25:52,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741945_1125 (size=6063) 2024-12-03T09:25:52,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741945_1125 (size=6063) 2024-12-03T09:25:52,609 INFO [M:0;699c9baf3d29:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef820256c4f94fb5b7c767f2186b302d 2024-12-03T09:25:52,627 DEBUG [M:0;699c9baf3d29:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa784ed96dfb451896825646eacc6371 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733217928628/Put/seqid=0 2024-12-03T09:25:52,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741946_1126 (size=7908) 2024-12-03T09:25:52,634 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741946_1126 (size=7908) 2024-12-03T09:25:52,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741946_1126 (size=7908) 2024-12-03T09:25:52,635 INFO [M:0;699c9baf3d29:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.18 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa784ed96dfb451896825646eacc6371 2024-12-03T09:25:52,641 INFO [M:0;699c9baf3d29:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa784ed96dfb451896825646eacc6371 2024-12-03T09:25:52,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,651 INFO [RS:1;699c9baf3d29:40727 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:25:52,651 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40727-0x1019a8282cb0002, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,651 INFO [RS:1;699c9baf3d29:40727 {}] regionserver.HRegionServer(1031): Exiting; stopping=699c9baf3d29,40727,1733217914150; zookeeper connection closed. 2024-12-03T09:25:52,651 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@48e18a63 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@48e18a63 2024-12-03T09:25:52,652 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T09:25:52,655 DEBUG [M:0;699c9baf3d29:33521 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb4755fec20c4d6486624d6bfa42378b is 69, key is 699c9baf3d29,40727,1733217914150/rs:state/1733217915376/Put/seqid=0 2024-12-03T09:25:52,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741947_1127 (size=5445) 2024-12-03T09:25:52,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741947_1127 (size=5445) 2024-12-03T09:25:52,662 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741947_1127 (size=5445) 2024-12-03T09:25:52,662 INFO [M:0;699c9baf3d29:33521 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb4755fec20c4d6486624d6bfa42378b 2024-12-03T09:25:52,667 INFO [M:0;699c9baf3d29:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb4755fec20c4d6486624d6bfa42378b 2024-12-03T09:25:52,668 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/ef820256c4f94fb5b7c767f2186b302d as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef820256c4f94fb5b7c767f2186b302d 2024-12-03T09:25:52,673 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/ef820256c4f94fb5b7c767f2186b302d, entries=14, sequenceid=207, filesize=5.9 K 2024-12-03T09:25:52,675 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa784ed96dfb451896825646eacc6371 as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa784ed96dfb451896825646eacc6371 2024-12-03T09:25:52,679 INFO [M:0;699c9baf3d29:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa784ed96dfb451896825646eacc6371 2024-12-03T09:25:52,679 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa784ed96dfb451896825646eacc6371, entries=21, sequenceid=207, filesize=7.7 K 2024-12-03T09:25:52,680 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/fb4755fec20c4d6486624d6bfa42378b as hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb4755fec20c4d6486624d6bfa42378b 2024-12-03T09:25:52,685 INFO [M:0;699c9baf3d29:33521 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fb4755fec20c4d6486624d6bfa42378b 2024-12-03T09:25:52,685 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:37427/user/jenkins/test-data/891e9f91-7a87-f828-1415-6c675848ff41/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/fb4755fec20c4d6486624d6bfa42378b, entries=3, sequenceid=207, filesize=5.3 K 2024-12-03T09:25:52,686 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=207, compaction requested=false 2024-12-03T09:25:52,687 INFO [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T09:25:52,687 DEBUG [M:0;699c9baf3d29:33521 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733217952585Disabling compacts and flushes for region at 1733217952585Disabling writes for close at 1733217952585Obtaining lock to block concurrent updates at 1733217952585Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733217952585Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85410, getHeapSize=105112, getOffHeapSize=0, getCellsCount=248 at 1733217952585Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733217952586 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733217952586Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733217952602 (+16 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733217952602Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733217952613 (+11 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733217952626 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733217952626Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733217952641 (+15 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733217952654 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733217952654Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a9ee942: reopening flushed file at 1733217952667 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@93346c0: reopening flushed file at 1733217952674 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60452b88: reopening flushed file at 1733217952679 (+5 ms)Finished flush of dataSize ~83.41 KB/85410, heapSize ~102.41 KB/104872, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=207, compaction requested=false at 1733217952686 (+7 ms)Writing region close event to WAL at 1733217952687 (+1 ms)Closed at 1733217952687 2024-12-03T09:25:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:41153 is added to blk_1073741830_1006 (size=69439) 2024-12-03T09:25:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36933 is added to blk_1073741830_1006 (size=69439) 2024-12-03T09:25:52,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34311 is added to blk_1073741830_1006 (size=69439) 2024-12-03T09:25:52,690 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T09:25:52,690 INFO [M:0;699c9baf3d29:33521 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T09:25:52,690 INFO [M:0;699c9baf3d29:33521 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:33521 2024-12-03T09:25:52,691 INFO [M:0;699c9baf3d29:33521 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T09:25:52,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,798 INFO [M:0;699c9baf3d29:33521 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T09:25:52,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:33521-0x1019a8282cb0000, quorum=127.0.0.1:57184, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T09:25:52,809 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217942269 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217942269 (inode 16688) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733217933054/wal.1733217942269 (inode 16688) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T09:25:52,812 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932921 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932921 (inode 16665) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733217932263/wal.1733217932921 (inode 16665) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T09:25:52,813 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733217923803/wal.1733217923901 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:52,815 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217943036 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217943036 (inode 16714) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733217942452/wal.1733217943036 (inode 16714) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T09:25:52,816 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733217943174/wal.1733217943284 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:52,816 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733217947612/wal.1733217947780 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:52,816 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733217917408/wal.1733217917474 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T09:25:52,818 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal.1733217917276 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal.1733217917276 (inode 16485) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733217917003/wal.1733217917276 (inode 16485) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T09:25:52,821 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal.1733217916755 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal.1733217916755 (inode 16462) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733217916520/wal.1733217916755 (inode 16462) Holder DFSClient_NONMAPREDUCE_1431312456_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor4.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T09:25:52,826 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@26f8cb79{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:52,827 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@10705e91{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:25:52,827 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:25:52,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@16c07ee3{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:25:52,828 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@4bbcb0d6{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,STOPPED} 2024-12-03T09:25:52,830 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T09:25:52,830 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T09:25:52,830 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T09:25:52,830 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1349158104-172.17.0.2-1733217908790 (Datanode Uuid f16e760b-d084-42b1-9f6c-0da663611f76) service to localhost/127.0.0.1:37427 2024-12-03T09:25:52,831 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data5/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,831 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data6/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,832 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T09:25:52,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4464725f{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:52,834 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@11808715{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:25:52,834 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:25:52,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@2f9065a5{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:25:52,834 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@243ebd61{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,STOPPED} 2024-12-03T09:25:52,835 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T09:25:52,835 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T09:25:52,835 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1349158104-172.17.0.2-1733217908790 (Datanode Uuid c42efa65-85a4-4907-ab2f-d747318fb4b5) service to localhost/127.0.0.1:37427 2024-12-03T09:25:52,835 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T09:25:52,836 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data3/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,836 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data4/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,836 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T09:25:52,840 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@17d1a8d5{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T09:25:52,840 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4afdb36e{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:25:52,840 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:25:52,840 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@fcc4f36{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:25:52,840 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@126e7def{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,STOPPED} 2024-12-03T09:25:52,841 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T09:25:52,841 WARN [BP-1349158104-172.17.0.2-1733217908790 heartbeating to localhost/127.0.0.1:37427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1349158104-172.17.0.2-1733217908790 (Datanode Uuid 6c8444e9-a104-4ae1-b929-4bed1af47d3d) service to localhost/127.0.0.1:37427 2024-12-03T09:25:52,842 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data2/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,842 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T09:25:52,842 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/cluster_d91079f5-a659-ba01-34d1-505f4bc12330/data/data1/current/BP-1349158104-172.17.0.2-1733217908790 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T09:25:52,842 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T09:25:52,842 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T09:25:52,847 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2c8b857b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T09:25:52,848 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@58a7ff93{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T09:25:52,848 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T09:25:52,848 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a6f93cf{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T09:25:52,848 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5dc5014{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/24e4d9bf-2a47-8603-80f9-a3ca75649d06/hadoop.log.dir/,STOPPED} 2024-12-03T09:25:52,857 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T09:25:52,903 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down