2024-11-13 03:31:41,208 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 03:31:41,222 main DEBUG Took 0.011539 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-13 03:31:41,222 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-13 03:31:41,223 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-13 03:31:41,224 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-13 03:31:41,225 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,234 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-13 03:31:41,248 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,250 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,251 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,251 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,252 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,252 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,253 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,254 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,255 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,256 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,257 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,258 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,258 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,259 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,260 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,260 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,260 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,261 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,261 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,262 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,262 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,263 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-13 03:31:41,263 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,264 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-13 03:31:41,266 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-13 03:31:41,267 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-13 03:31:41,269 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-13 03:31:41,269 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-13 03:31:41,271 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-13 03:31:41,272 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-13 03:31:41,280 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-13 03:31:41,283 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-13 03:31:41,285 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-13 03:31:41,285 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-13 03:31:41,286 main DEBUG createAppenders(={Console}) 2024-11-13 03:31:41,286 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f initialized 2024-11-13 03:31:41,287 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f 2024-11-13 03:31:41,287 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@210ab13f OK. 2024-11-13 03:31:41,288 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-13 03:31:41,288 main DEBUG OutputStream closed 2024-11-13 03:31:41,288 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-13 03:31:41,289 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-13 03:31:41,289 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@61001b64 OK 2024-11-13 03:31:41,373 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-13 03:31:41,376 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-13 03:31:41,377 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-13 03:31:41,379 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-13 03:31:41,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-13 03:31:41,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-13 03:31:41,380 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-13 03:31:41,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-13 03:31:41,381 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-13 03:31:41,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-13 03:31:41,382 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-13 03:31:41,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-13 03:31:41,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-13 03:31:41,383 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-13 03:31:41,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-13 03:31:41,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-13 03:31:41,384 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-13 03:31:41,385 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-13 03:31:41,388 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13 03:31:41,388 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@4ed5eb72) with optional ClassLoader: null 2024-11-13 03:31:41,388 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-13 03:31:41,389 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@4ed5eb72] started OK. 2024-11-13T03:31:41,631 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a 2024-11-13 03:31:41,633 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-13 03:31:41,633 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-13T03:31:41,641 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-11-13T03:31:41,673 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-13T03:31:41,718 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-13T03:31:41,718 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-13T03:31:41,731 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-13T03:31:41,746 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d, deleteOnExit=true 2024-11-13T03:31:41,746 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-13T03:31:41,747 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/test.cache.data in system properties and HBase conf 2024-11-13T03:31:41,748 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.tmp.dir in system properties and HBase conf 2024-11-13T03:31:41,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir in system properties and HBase conf 2024-11-13T03:31:41,749 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-13T03:31:41,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-13T03:31:41,750 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-13T03:31:41,822 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-13T03:31:41,913 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-13T03:31:41,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:31:41,919 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-13T03:31:41,920 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-13T03:31:41,921 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:31:41,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-13T03:31:41,922 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-13T03:31:41,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-13T03:31:41,923 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:31:41,924 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-13T03:31:41,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/nfs.dump.dir in system properties and HBase conf 2024-11-13T03:31:41,925 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/java.io.tmpdir in system properties and HBase conf 2024-11-13T03:31:41,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-13T03:31:41,926 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-13T03:31:41,927 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-13T03:31:43,036 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-13T03:31:43,102 INFO [Time-limited test {}] log.Log(170): Logging initialized @2556ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-13T03:31:43,165 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:31:43,223 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:31:43,250 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:31:43,250 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:31:43,251 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:31:43,263 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:31:43,266 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:31:43,268 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:31:43,433 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@398d2d49{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/java.io.tmpdir/jetty-localhost-46697-hadoop-hdfs-3_4_1-tests_jar-_-any-8997490372224924950/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:31:43,439 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:46697} 2024-11-13T03:31:43,439 INFO [Time-limited test {}] server.Server(415): Started @2894ms 2024-11-13T03:31:44,025 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:31:44,033 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:31:44,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:31:44,034 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:31:44,034 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:31:44,035 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:31:44,036 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:31:44,133 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@54387323{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/java.io.tmpdir/jetty-localhost-34747-hadoop-hdfs-3_4_1-tests_jar-_-any-16503781516311754887/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:31:44,134 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:34747} 2024-11-13T03:31:44,134 INFO [Time-limited test {}] server.Server(415): Started @3588ms 2024-11-13T03:31:44,180 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:31:44,286 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:31:44,293 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:31:44,295 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:31:44,295 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:31:44,295 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-13T03:31:44,296 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:31:44,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:31:44,424 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@7e9308b2{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/java.io.tmpdir/jetty-localhost-43175-hadoop-hdfs-3_4_1-tests_jar-_-any-16637796310798868441/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:31:44,425 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:43175} 2024-11-13T03:31:44,425 INFO [Time-limited test {}] server.Server(415): Started @3880ms 2024-11-13T03:31:44,429 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:31:44,465 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-13T03:31:44,471 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-13T03:31:44,473 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-13T03:31:44,474 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-13T03:31:44,474 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-13T03:31:44,475 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,AVAILABLE} 2024-11-13T03:31:44,476 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-13T03:31:44,583 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@3217543c{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/java.io.tmpdir/jetty-localhost-34739-hadoop-hdfs-3_4_1-tests_jar-_-any-1686610823599548351/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:31:44,584 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:34739} 2024-11-13T03:31:44,584 INFO [Time-limited test {}] server.Server(415): Started @4039ms 2024-11-13T03:31:44,587 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-13T03:31:45,853 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data3/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,853 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data2/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,853 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data1/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,853 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data4/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,889 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:31:45,889 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:31:45,898 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data5/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,901 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data6/current/BP-648661827-172.17.0.3-1731468702443/current, will proceed with Du for space computation calculation, 2024-11-13T03:31:45,937 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-13T03:31:45,953 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd257f8dab28fd3b1 with lease ID 0x5074fcf57f2de092: Processing first storage report for DS-0355b42a-550a-41e6-b90d-fde8ff4f3463 from datanode DatanodeRegistration(127.0.0.1:43451, datanodeUuid=01ac62a3-65ab-469f-ad86-8fba4fe12067, infoPort=34497, infoSecurePort=0, ipcPort=45567, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd257f8dab28fd3b1 with lease ID 0x5074fcf57f2de092: from storage DS-0355b42a-550a-41e6-b90d-fde8ff4f3463 node DatanodeRegistration(127.0.0.1:43451, datanodeUuid=01ac62a3-65ab-469f-ad86-8fba4fe12067, infoPort=34497, infoSecurePort=0, ipcPort=45567, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-13T03:31:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xecc7f30db63702a2 with lease ID 0x5074fcf57f2de093: Processing first storage report for DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e from datanode DatanodeRegistration(127.0.0.1:43285, datanodeUuid=a477af8c-4f94-4dc1-a388-b8b6c03e8704, infoPort=41907, infoSecurePort=0, ipcPort=41461, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xecc7f30db63702a2 with lease ID 0x5074fcf57f2de093: from storage DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e node DatanodeRegistration(127.0.0.1:43285, datanodeUuid=a477af8c-4f94-4dc1-a388-b8b6c03e8704, infoPort=41907, infoSecurePort=0, ipcPort=41461, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:31:45,954 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96805d556cb2731e with lease ID 0x5074fcf57f2de094: Processing first storage report for DS-75a97f47-f4da-4c5c-ae35-34f12a772a01 from datanode DatanodeRegistration(127.0.0.1:37801, datanodeUuid=b4309cbc-40ad-4408-a07a-b48671724b7a, infoPort=40391, infoSecurePort=0, ipcPort=35447, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96805d556cb2731e with lease ID 0x5074fcf57f2de094: from storage DS-75a97f47-f4da-4c5c-ae35-34f12a772a01 node DatanodeRegistration(127.0.0.1:37801, datanodeUuid=b4309cbc-40ad-4408-a07a-b48671724b7a, infoPort=40391, infoSecurePort=0, ipcPort=35447, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xd257f8dab28fd3b1 with lease ID 0x5074fcf57f2de092: Processing first storage report for DS-5f882da7-2867-4559-bef1-1e9bb7bdb203 from datanode DatanodeRegistration(127.0.0.1:43451, datanodeUuid=01ac62a3-65ab-469f-ad86-8fba4fe12067, infoPort=34497, infoSecurePort=0, ipcPort=45567, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xd257f8dab28fd3b1 with lease ID 0x5074fcf57f2de092: from storage DS-5f882da7-2867-4559-bef1-1e9bb7bdb203 node DatanodeRegistration(127.0.0.1:43451, datanodeUuid=01ac62a3-65ab-469f-ad86-8fba4fe12067, infoPort=34497, infoSecurePort=0, ipcPort=45567, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xecc7f30db63702a2 with lease ID 0x5074fcf57f2de093: Processing first storage report for DS-e90ad696-299c-40cd-aef1-0a175235922d from datanode DatanodeRegistration(127.0.0.1:43285, datanodeUuid=a477af8c-4f94-4dc1-a388-b8b6c03e8704, infoPort=41907, infoSecurePort=0, ipcPort=41461, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xecc7f30db63702a2 with lease ID 0x5074fcf57f2de093: from storage DS-e90ad696-299c-40cd-aef1-0a175235922d node DatanodeRegistration(127.0.0.1:43285, datanodeUuid=a477af8c-4f94-4dc1-a388-b8b6c03e8704, infoPort=41907, infoSecurePort=0, ipcPort=41461, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:31:45,955 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x96805d556cb2731e with lease ID 0x5074fcf57f2de094: Processing first storage report for DS-20a13f54-b6ed-40de-bd3a-41b233f30e6b from datanode DatanodeRegistration(127.0.0.1:37801, datanodeUuid=b4309cbc-40ad-4408-a07a-b48671724b7a, infoPort=40391, infoSecurePort=0, ipcPort=35447, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443) 2024-11-13T03:31:45,956 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x96805d556cb2731e with lease ID 0x5074fcf57f2de094: from storage DS-20a13f54-b6ed-40de-bd3a-41b233f30e6b node DatanodeRegistration(127.0.0.1:37801, datanodeUuid=b4309cbc-40ad-4408-a07a-b48671724b7a, infoPort=40391, infoSecurePort=0, ipcPort=35447, storageInfo=lv=-57;cid=testClusterID;nsid=70493818;c=1731468702443), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-13T03:31:46,025 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a 2024-11-13T03:31:46,110 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/zookeeper_0, clientPort=58534, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-13T03:31:46,118 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=58534 2024-11-13T03:31:46,127 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:46,129 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:46,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:31:46,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:31:46,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741825_1001 (size=7) 2024-11-13T03:31:46,719 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda with version=8 2024-11-13T03:31:46,720 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/hbase-staging 2024-11-13T03:31:47,019 INFO [Time-limited test {}] client.ConnectionUtils(128): master/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:31:47,027 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,028 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,032 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:31:47,032 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,033 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:31:47,162 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-13T03:31:47,218 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-13T03:31:47,227 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-13T03:31:47,230 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:31:47,254 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 90042 (auto-detected) 2024-11-13T03:31:47,255 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-13T03:31:47,272 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41967 2024-11-13T03:31:47,292 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:41967 connecting to ZooKeeper ensemble=127.0.0.1:58534 2024-11-13T03:31:47,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:419670x0, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:31:47,419 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:41967-0x101323fbcd90000 connected 2024-11-13T03:31:47,520 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,524 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,534 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:31:47,538 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda, hbase.cluster.distributed=false 2024-11-13T03:31:47,558 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:31:47,562 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41967 2024-11-13T03:31:47,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41967 2024-11-13T03:31:47,563 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41967 2024-11-13T03:31:47,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41967 2024-11-13T03:31:47,564 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41967 2024-11-13T03:31:47,659 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:31:47,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,660 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,660 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:31:47,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,661 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:31:47,663 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:31:47,666 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:31:47,666 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:43461 2024-11-13T03:31:47,668 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:43461 connecting to ZooKeeper ensemble=127.0.0.1:58534 2024-11-13T03:31:47,669 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:434610x0, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:31:47,686 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:434610x0, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:31:47,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:43461-0x101323fbcd90001 connected 2024-11-13T03:31:47,689 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:31:47,696 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:31:47,698 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:31:47,703 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:31:47,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=43461 2024-11-13T03:31:47,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=43461 2024-11-13T03:31:47,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=43461 2024-11-13T03:31:47,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=43461 2024-11-13T03:31:47,705 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=43461 2024-11-13T03:31:47,723 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:31:47,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,724 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:31:47,724 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,725 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:31:47,725 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:31:47,725 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:31:47,726 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:41763 2024-11-13T03:31:47,728 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:41763 connecting to ZooKeeper ensemble=127.0.0.1:58534 2024-11-13T03:31:47,729 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,732 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:417630x0, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:31:47,749 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:417630x0, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:31:47,749 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41763-0x101323fbcd90002 connected 2024-11-13T03:31:47,750 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:31:47,751 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:31:47,752 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:31:47,754 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:31:47,755 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41763 2024-11-13T03:31:47,756 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41763 2024-11-13T03:31:47,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41763 2024-11-13T03:31:47,758 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41763 2024-11-13T03:31:47,759 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41763 2024-11-13T03:31:47,777 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/8394b182524b:0 server-side Connection retries=45 2024-11-13T03:31:47,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,777 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,778 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-13T03:31:47,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-13T03:31:47,778 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-13T03:31:47,778 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-13T03:31:47,779 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-13T03:31:47,780 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:40261 2024-11-13T03:31:47,782 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:40261 connecting to ZooKeeper ensemble=127.0.0.1:58534 2024-11-13T03:31:47,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,786 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:47,801 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:402610x0, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-13T03:31:47,801 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:402610x0, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:31:47,801 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40261-0x101323fbcd90003 connected 2024-11-13T03:31:47,802 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-13T03:31:47,803 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-13T03:31:47,804 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-13T03:31:47,807 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-13T03:31:47,807 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-13T03:31:47,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40261 2024-11-13T03:31:47,808 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40261 2024-11-13T03:31:47,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-13T03:31:47,811 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40261 2024-11-13T03:31:47,826 DEBUG [M:0;8394b182524b:41967 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;8394b182524b:41967 2024-11-13T03:31:47,827 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/8394b182524b,41967,1731468706868 2024-11-13T03:31:47,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,843 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,845 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/8394b182524b,41967,1731468706868 2024-11-13T03:31:47,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:31:47,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:31:47,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-13T03:31:47,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:47,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:47,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:47,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:47,875 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-13T03:31:47,876 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/8394b182524b,41967,1731468706868 from backup master directory 2024-11-13T03:31:47,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/8394b182524b,41967,1731468706868 2024-11-13T03:31:47,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,885 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,886 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-13T03:31:47,886 WARN [master/8394b182524b:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:31:47,887 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=8394b182524b,41967,1731468706868 2024-11-13T03:31:47,890 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-13T03:31:47,891 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-13T03:31:47,944 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/hbase.id] with ID: 1ce67cc5-da14-493a-907b-407a287aeb9e 2024-11-13T03:31:47,944 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/.tmp/hbase.id 2024-11-13T03:31:47,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:31:47,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:31:47,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741826_1002 (size=42) 2024-11-13T03:31:47,960 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/.tmp/hbase.id]:[hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/hbase.id] 2024-11-13T03:31:48,003 INFO [master/8394b182524b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-13T03:31:48,007 INFO [master/8394b182524b:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-13T03:31:48,024 INFO [master/8394b182524b:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-13T03:31:48,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,032 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:31:48,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:31:48,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741827_1003 (size=196) 2024-11-13T03:31:48,065 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:31:48,066 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-13T03:31:48,071 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:48,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:31:48,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:31:48,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741828_1004 (size=1189) 2024-11-13T03:31:48,119 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store 2024-11-13T03:31:48,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:31:48,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:31:48,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741829_1005 (size=34) 2024-11-13T03:31:48,145 INFO [master/8394b182524b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-13T03:31:48,148 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:48,149 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:31:48,149 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:31:48,150 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:31:48,151 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:31:48,151 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:31:48,151 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:31:48,153 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731468708149Disabling compacts and flushes for region at 1731468708149Disabling writes for close at 1731468708151 (+2 ms)Writing region close event to WAL at 1731468708151Closed at 1731468708151 2024-11-13T03:31:48,155 WARN [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/.initializing 2024-11-13T03:31:48,155 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/WALs/8394b182524b,41967,1731468706868 2024-11-13T03:31:48,163 INFO [master/8394b182524b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:48,177 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C41967%2C1731468706868, suffix=, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/WALs/8394b182524b,41967,1731468706868, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/oldWALs, maxLogs=10 2024-11-13T03:31:48,204 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/WALs/8394b182524b,41967,1731468706868/8394b182524b%2C41967%2C1731468706868.1731468708181, exclude list is [], retry=0 2024-11-13T03:31:48,221 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:48,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:48,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:48,225 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:48,227 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-13T03:31:48,263 INFO [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/WALs/8394b182524b,41967,1731468706868/8394b182524b%2C41967%2C1731468706868.1731468708181 2024-11-13T03:31:48,264 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:48,265 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:48,265 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:48,269 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,270 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,307 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,330 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-13T03:31:48,333 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:48,335 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:48,336 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,339 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-13T03:31:48,339 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:48,340 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:48,341 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,343 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-13T03:31:48,344 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:48,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:48,345 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,348 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-13T03:31:48,348 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:48,349 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:48,350 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,354 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,356 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,361 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,362 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,366 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:31:48,369 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-13T03:31:48,374 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:48,375 INFO [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63508554, jitterRate=-0.053648799657821655}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:31:48,383 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731468708282Initializing all the Stores at 1731468708284 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468708284Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468708285 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468708285Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468708285Cleaning up temporary data from old regions at 1731468708362 (+77 ms)Region opened successfully at 1731468708382 (+20 ms) 2024-11-13T03:31:48,384 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-13T03:31:48,418 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a60ed02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:31:48,446 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-13T03:31:48,455 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-13T03:31:48,455 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-13T03:31:48,458 INFO [master/8394b182524b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-13T03:31:48,460 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-13T03:31:48,464 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-13T03:31:48,464 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-13T03:31:48,494 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-13T03:31:48,504 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-13T03:31:48,650 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-13T03:31:48,658 INFO [master/8394b182524b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-13T03:31:48,660 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-13T03:31:48,673 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-13T03:31:48,676 INFO [master/8394b182524b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-13T03:31:48,680 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-13T03:31:48,684 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-13T03:31:48,686 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-13T03:31:48,695 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-13T03:31:48,716 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-13T03:31:48,726 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-13T03:31:48,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:31:48,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:31:48,737 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:31:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-13T03:31:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,738 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,743 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=8394b182524b,41967,1731468706868, sessionid=0x101323fbcd90000, setting cluster-up flag (Was=false) 2024-11-13T03:31:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,769 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,801 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-13T03:31:48,806 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8394b182524b,41967,1731468706868 2024-11-13T03:31:48,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:48,863 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-13T03:31:48,866 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=8394b182524b,41967,1731468706868 2024-11-13T03:31:48,873 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-13T03:31:48,915 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(746): ClusterId : 1ce67cc5-da14-493a-907b-407a287aeb9e 2024-11-13T03:31:48,915 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(746): ClusterId : 1ce67cc5-da14-493a-907b-407a287aeb9e 2024-11-13T03:31:48,918 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(746): ClusterId : 1ce67cc5-da14-493a-907b-407a287aeb9e 2024-11-13T03:31:48,918 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:31:48,918 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:31:48,918 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-13T03:31:48,929 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:31:48,929 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:31:48,929 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-13T03:31:48,929 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:31:48,929 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:31:48,929 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-13T03:31:48,938 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:31:48,938 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:31:48,938 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-13T03:31:48,939 DEBUG [RS:1;8394b182524b:41763 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@49a4e112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:31:48,939 DEBUG [RS:2;8394b182524b:40261 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e3596f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:31:48,939 DEBUG [RS:0;8394b182524b:43461 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72dbb6a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=8394b182524b/172.17.0.3:0 2024-11-13T03:31:48,945 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-13T03:31:48,955 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;8394b182524b:41763 2024-11-13T03:31:48,957 INFO [master/8394b182524b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-13T03:31:48,958 INFO [RS:1;8394b182524b:41763 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:31:48,958 INFO [RS:1;8394b182524b:41763 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:31:48,958 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:31:48,959 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;8394b182524b:43461 2024-11-13T03:31:48,959 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;8394b182524b:40261 2024-11-13T03:31:48,959 INFO [RS:2;8394b182524b:40261 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:31:48,959 INFO [RS:0;8394b182524b:43461 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-13T03:31:48,959 INFO [RS:2;8394b182524b:40261 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:31:48,959 INFO [RS:0;8394b182524b:43461 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-13T03:31:48,959 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:31:48,959 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-13T03:31:48,961 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,41967,1731468706868 with port=41763, startcode=1731468707723 2024-11-13T03:31:48,961 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,41967,1731468706868 with port=43461, startcode=1731468707627 2024-11-13T03:31:48,961 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(2659): reportForDuty to master=8394b182524b,41967,1731468706868 with port=40261, startcode=1731468707776 2024-11-13T03:31:48,965 INFO [master/8394b182524b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-13T03:31:48,972 DEBUG [RS:1;8394b182524b:41763 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:31:48,972 DEBUG [RS:2;8394b182524b:40261 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:31:48,972 DEBUG [RS:0;8394b182524b:43461 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-13T03:31:48,970 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 8394b182524b,41967,1731468706868 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-13T03:31:48,977 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:31:48,977 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:31:48,977 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:31:48,977 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/8394b182524b:0, corePoolSize=5, maxPoolSize=5 2024-11-13T03:31:48,978 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/8394b182524b:0, corePoolSize=10, maxPoolSize=10 2024-11-13T03:31:48,978 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:48,978 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:31:48,978 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:48,983 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731468738983 2024-11-13T03:31:48,985 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-13T03:31:48,986 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-13T03:31:48,990 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-13T03:31:48,991 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-13T03:31:48,991 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-13T03:31:48,991 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-13T03:31:48,992 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:48,996 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-13T03:31:48,997 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-13T03:31:48,998 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-13T03:31:49,000 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-13T03:31:49,001 INFO [master/8394b182524b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-13T03:31:49,001 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:31:49,001 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-13T03:31:49,003 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468709002,5,FailOnTimeoutGroup] 2024-11-13T03:31:49,003 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468709003,5,FailOnTimeoutGroup] 2024-11-13T03:31:49,003 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,004 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-13T03:31:49,005 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,006 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,007 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:47911, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:31:49,007 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57199, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:31:49,007 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48533, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-11-13T03:31:49,009 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,009 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-13T03:31:49,014 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,43461,1731468707627 2024-11-13T03:31:49,016 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(517): Registering regionserver=8394b182524b,43461,1731468707627 2024-11-13T03:31:49,037 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,40261,1731468707776 2024-11-13T03:31:49,037 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(517): Registering regionserver=8394b182524b,40261,1731468707776 2024-11-13T03:31:49,041 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda 2024-11-13T03:31:49,041 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33705 2024-11-13T03:31:49,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:31:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:31:49,042 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:31:49,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741831_1007 (size=1321) 2024-11-13T03:31:49,043 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 8394b182524b,41763,1731468707723 2024-11-13T03:31:49,043 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=41967 {}] master.ServerManager(517): Registering regionserver=8394b182524b,41763,1731468707723 2024-11-13T03:31:49,043 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda 2024-11-13T03:31:49,043 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33705 2024-11-13T03:31:49,044 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:31:49,044 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-13T03:31:49,045 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda 2024-11-13T03:31:49,048 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda 2024-11-13T03:31:49,048 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:33705 2024-11-13T03:31:49,048 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-13T03:31:49,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:31:49,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:31:49,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:31:49,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741832_1008 (size=32) 2024-11-13T03:31:49,061 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:49,063 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:31:49,065 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:31:49,066 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,066 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,067 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:31:49,069 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:31:49,069 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,069 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,070 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:31:49,072 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:31:49,072 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,073 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,074 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:31:49,077 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:31:49,077 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,078 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,078 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:31:49,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:31:49,080 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:31:49,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:31:49,083 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:31:49,084 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:31:49,086 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:31:49,090 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:49,091 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59135662, jitterRate=-0.11880996823310852}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:31:49,094 DEBUG [RS:0;8394b182524b:43461 {}] zookeeper.ZKUtil(111): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,43461,1731468707627 2024-11-13T03:31:49,094 WARN [RS:0;8394b182524b:43461 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:31:49,094 DEBUG [RS:2;8394b182524b:40261 {}] zookeeper.ZKUtil(111): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,40261,1731468707776 2024-11-13T03:31:49,094 INFO [RS:0;8394b182524b:43461 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:49,094 DEBUG [RS:1;8394b182524b:41763 {}] zookeeper.ZKUtil(111): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/8394b182524b,41763,1731468707723 2024-11-13T03:31:49,094 WARN [RS:2;8394b182524b:40261 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:31:49,094 WARN [RS:1;8394b182524b:41763 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-13T03:31:49,095 INFO [RS:2;8394b182524b:40261 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:49,095 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627 2024-11-13T03:31:49,095 INFO [RS:1;8394b182524b:41763 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:49,095 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776 2024-11-13T03:31:49,095 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,41763,1731468707723 2024-11-13T03:31:49,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731468709061Initializing all the Stores at 1731468709063 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709063Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709063Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468709063Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709063Cleaning up temporary data from old regions at 1731468709083 (+20 ms)Region opened successfully at 1731468709096 (+13 ms) 2024-11-13T03:31:49,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:31:49,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,43461,1731468707627] 2024-11-13T03:31:49,096 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:31:49,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,41763,1731468707723] 2024-11-13T03:31:49,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:31:49,096 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [8394b182524b,40261,1731468707776] 2024-11-13T03:31:49,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:31:49,096 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:31:49,099 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:31:49,099 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731468709096Disabling compacts and flushes for region at 1731468709096Disabling writes for close at 1731468709096Writing region close event to WAL at 1731468709098 (+2 ms)Closed at 1731468709099 (+1 ms) 2024-11-13T03:31:49,104 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:31:49,104 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-13T03:31:49,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T03:31:49,119 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:31:49,123 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-13T03:31:49,123 INFO [RS:1;8394b182524b:41763 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:31:49,123 INFO [RS:0;8394b182524b:43461 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:31:49,123 INFO [RS:2;8394b182524b:40261 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-13T03:31:49,138 INFO [RS:1;8394b182524b:41763 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:31:49,138 INFO [RS:2;8394b182524b:40261 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:31:49,139 INFO [RS:0;8394b182524b:43461 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-13T03:31:49,144 INFO [RS:2;8394b182524b:40261 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:31:49,144 INFO [RS:0;8394b182524b:43461 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:31:49,144 INFO [RS:1;8394b182524b:41763 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:31:49,144 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,144 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,144 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,145 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:31:49,145 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:31:49,146 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-13T03:31:49,151 INFO [RS:2;8394b182524b:40261 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:31:49,151 INFO [RS:1;8394b182524b:41763 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:31:49,151 INFO [RS:0;8394b182524b:43461 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-13T03:31:49,152 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,152 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,152 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:31:49,153 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,153 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0, corePoolSize=2, maxPoolSize=2 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,154 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/8394b182524b:0, corePoolSize=1, maxPoolSize=1 2024-11-13T03:31:49,154 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,155 DEBUG [RS:1;8394b182524b:41763 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,155 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,155 DEBUG [RS:0;8394b182524b:43461 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,155 DEBUG [RS:2;8394b182524b:40261 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/8394b182524b:0, corePoolSize=3, maxPoolSize=3 2024-11-13T03:31:49,156 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,156 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,156 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,156 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,157 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,157 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,43461,1731468707627-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,160 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,161 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41763,1731468707723-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:31:49,161 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,161 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,40261,1731468707776-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:31:49,179 INFO [RS:2;8394b182524b:40261 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:31:49,181 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,40261,1731468707776-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,181 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,181 INFO [RS:2;8394b182524b:40261 {}] regionserver.Replication(171): 8394b182524b,40261,1731468707776 started 2024-11-13T03:31:49,182 INFO [RS:0;8394b182524b:43461 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:31:49,182 INFO [RS:1;8394b182524b:41763 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-13T03:31:49,182 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,43461,1731468707627-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,182 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41763,1731468707723-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,182 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,182 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,182 INFO [RS:0;8394b182524b:43461 {}] regionserver.Replication(171): 8394b182524b,43461,1731468707627 started 2024-11-13T03:31:49,182 INFO [RS:1;8394b182524b:41763 {}] regionserver.Replication(171): 8394b182524b,41763,1731468707723 started 2024-11-13T03:31:49,208 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,208 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,208 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:49,208 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,43461,1731468707627, RpcServer on 8394b182524b/172.17.0.3:43461, sessionid=0x101323fbcd90001 2024-11-13T03:31:49,208 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,41763,1731468707723, RpcServer on 8394b182524b/172.17.0.3:41763, sessionid=0x101323fbcd90002 2024-11-13T03:31:49,208 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1482): Serving as 8394b182524b,40261,1731468707776, RpcServer on 8394b182524b/172.17.0.3:40261, sessionid=0x101323fbcd90003 2024-11-13T03:31:49,209 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:31:49,209 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:31:49,209 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-13T03:31:49,209 DEBUG [RS:0;8394b182524b:43461 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,43461,1731468707627 2024-11-13T03:31:49,209 DEBUG [RS:2;8394b182524b:40261 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,40261,1731468707776 2024-11-13T03:31:49,209 DEBUG [RS:1;8394b182524b:41763 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 8394b182524b,41763,1731468707723 2024-11-13T03:31:49,210 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,41763,1731468707723' 2024-11-13T03:31:49,210 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,40261,1731468707776' 2024-11-13T03:31:49,210 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,43461,1731468707627' 2024-11-13T03:31:49,210 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:31:49,210 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:31:49,210 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-13T03:31:49,211 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:31:49,211 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:31:49,211 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-13T03:31:49,211 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:31:49,211 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:31:49,211 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-13T03:31:49,212 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:31:49,212 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:31:49,212 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-13T03:31:49,212 DEBUG [RS:0;8394b182524b:43461 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,43461,1731468707627 2024-11-13T03:31:49,212 DEBUG [RS:1;8394b182524b:41763 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,41763,1731468707723 2024-11-13T03:31:49,212 DEBUG [RS:2;8394b182524b:40261 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 8394b182524b,40261,1731468707776 2024-11-13T03:31:49,212 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,43461,1731468707627' 2024-11-13T03:31:49,212 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,41763,1731468707723' 2024-11-13T03:31:49,212 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '8394b182524b,40261,1731468707776' 2024-11-13T03:31:49,212 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:31:49,212 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:31:49,212 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-13T03:31:49,212 DEBUG [RS:0;8394b182524b:43461 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:31:49,212 DEBUG [RS:1;8394b182524b:41763 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:31:49,212 DEBUG [RS:2;8394b182524b:40261 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-13T03:31:49,213 DEBUG [RS:0;8394b182524b:43461 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:31:49,213 DEBUG [RS:2;8394b182524b:40261 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:31:49,213 INFO [RS:0;8394b182524b:43461 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:31:49,213 INFO [RS:2;8394b182524b:40261 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:31:49,213 DEBUG [RS:1;8394b182524b:41763 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-13T03:31:49,213 INFO [RS:2;8394b182524b:40261 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:31:49,213 INFO [RS:0;8394b182524b:43461 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:31:49,213 INFO [RS:1;8394b182524b:41763 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-13T03:31:49,213 INFO [RS:1;8394b182524b:41763 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-13T03:31:49,274 WARN [8394b182524b:41967 {}] assignment.AssignmentManager(2443): No servers available; cannot place 1 unassigned regions. 2024-11-13T03:31:49,318 INFO [RS:2;8394b182524b:40261 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:49,318 INFO [RS:0;8394b182524b:43461 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:49,318 INFO [RS:1;8394b182524b:41763 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:49,322 INFO [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C43461%2C1731468707627, suffix=, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs, maxLogs=32 2024-11-13T03:31:49,322 INFO [RS:1;8394b182524b:41763 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C41763%2C1731468707723, suffix=, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,41763,1731468707723, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs, maxLogs=32 2024-11-13T03:31:49,322 INFO [RS:2;8394b182524b:40261 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C40261%2C1731468707776, suffix=, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs, maxLogs=32 2024-11-13T03:31:49,339 DEBUG [RS:0;8394b182524b:43461 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627/8394b182524b%2C43461%2C1731468707627.1731468709325, exclude list is [], retry=0 2024-11-13T03:31:49,339 DEBUG [RS:1;8394b182524b:41763 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,41763,1731468707723/8394b182524b%2C41763%2C1731468707723.1731468709325, exclude list is [], retry=0 2024-11-13T03:31:49,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:49,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:49,344 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:49,345 DEBUG [RS:2;8394b182524b:40261 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776/8394b182524b%2C40261%2C1731468707776.1731468709326, exclude list is [], retry=0 2024-11-13T03:31:49,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:49,345 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:49,346 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:49,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:49,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:49,351 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:49,353 INFO [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627/8394b182524b%2C43461%2C1731468707627.1731468709325 2024-11-13T03:31:49,355 INFO [RS:1;8394b182524b:41763 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,41763,1731468707723/8394b182524b%2C41763%2C1731468707723.1731468709325 2024-11-13T03:31:49,358 DEBUG [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:31:49,358 DEBUG [RS:1;8394b182524b:41763 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:49,359 INFO [RS:2;8394b182524b:40261 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776/8394b182524b%2C40261%2C1731468707776.1731468709326 2024-11-13T03:31:49,360 DEBUG [RS:2;8394b182524b:40261 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:49,529 DEBUG [8394b182524b:41967 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-11-13T03:31:49,539 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T03:31:49,545 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:31:49,545 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:31:49,545 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T03:31:49,545 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:31:49,552 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:31:49,558 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,40261,1731468707776, state=OPENING 2024-11-13T03:31:49,611 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-13T03:31:49,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:49,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:49,622 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:49,621 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:31:49,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,623 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,624 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,626 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:31:49,629 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,40261,1731468707776}] 2024-11-13T03:31:49,808 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:31:49,809 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39715, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:31:49,820 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T03:31:49,820 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:49,821 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T03:31:49,824 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C40261%2C1731468707776.meta, suffix=.meta, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs, maxLogs=32 2024-11-13T03:31:49,841 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, exclude list is [], retry=0 2024-11-13T03:31:49,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:49,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:49,845 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:49,848 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta 2024-11-13T03:31:49,848 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:49,849 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:49,850 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T03:31:49,852 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T03:31:49,856 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T03:31:49,860 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T03:31:49,860 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:49,861 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T03:31:49,861 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T03:31:49,864 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:31:49,866 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:31:49,866 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,867 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:31:49,868 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:31:49,868 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,869 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:31:49,871 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:31:49,871 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,871 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,872 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:31:49,873 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:31:49,873 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:49,874 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:31:49,874 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:31:49,876 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:31:49,879 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:31:49,882 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:31:49,882 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:31:49,883 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:31:49,887 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:31:49,888 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67438425, jitterRate=0.0049108415842056274}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:31:49,889 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T03:31:49,890 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731468709861Writing region info on filesystem at 1731468709862 (+1 ms)Initializing all the Stores at 1731468709863 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709864 (+1 ms)Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709864Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468709864Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468709864Cleaning up temporary data from old regions at 1731468709882 (+18 ms)Running coprocessor post-open hooks at 1731468709889 (+7 ms)Region opened successfully at 1731468709890 (+1 ms) 2024-11-13T03:31:49,898 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731468709800 2024-11-13T03:31:49,910 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T03:31:49,911 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T03:31:49,912 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:31:49,915 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,40261,1731468707776, state=OPEN 2024-11-13T03:31:49,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:31:49,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:31:49,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:31:49,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:31:49,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,970 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:31:49,971 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8394b182524b,40261,1731468707776 2024-11-13T03:31:49,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-13T03:31:49,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,40261,1731468707776 in 343 msec 2024-11-13T03:31:49,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-13T03:31:49,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 872 msec 2024-11-13T03:31:49,987 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-13T03:31:49,987 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-13T03:31:50,008 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:31:50,009 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1] 2024-11-13T03:31:50,055 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:31:50,058 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50011, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:31:50,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.1760 sec 2024-11-13T03:31:50,080 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731468710079, completionTime=-1 2024-11-13T03:31:50,082 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-11-13T03:31:50,082 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-13T03:31:50,111 INFO [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=3 2024-11-13T03:31:50,112 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731468770112 2024-11-13T03:31:50,112 INFO [master/8394b182524b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731468830112 2024-11-13T03:31:50,112 INFO [master/8394b182524b:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 29 msec 2024-11-13T03:31:50,113 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-11-13T03:31:50,119 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,120 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,120 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,121 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-8394b182524b:41967, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,121 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,122 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,128 DEBUG [master/8394b182524b:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-13T03:31:50,149 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 2.262sec 2024-11-13T03:31:50,151 INFO [master/8394b182524b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-13T03:31:50,152 INFO [master/8394b182524b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-13T03:31:50,153 INFO [master/8394b182524b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-13T03:31:50,153 INFO [master/8394b182524b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-13T03:31:50,153 INFO [master/8394b182524b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-13T03:31:50,154 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-13T03:31:50,154 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-13T03:31:50,159 DEBUG [master/8394b182524b:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-13T03:31:50,160 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-13T03:31:50,161 INFO [master/8394b182524b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=8394b182524b,41967,1731468706868-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-13T03:31:50,243 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68674fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:31:50,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 8394b182524b,41967,-1 for getting cluster id 2024-11-13T03:31:50,246 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-13T03:31:50,254 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1ce67cc5-da14-493a-907b-407a287aeb9e' 2024-11-13T03:31:50,256 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-13T03:31:50,256 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1ce67cc5-da14-493a-907b-407a287aeb9e" 2024-11-13T03:31:50,257 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e9f3557, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:31:50,257 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [8394b182524b,41967,-1] 2024-11-13T03:31:50,259 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-13T03:31:50,261 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:31:50,262 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42200, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-13T03:31:50,264 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb9e3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-13T03:31:50,265 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:31:50,271 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1] 2024-11-13T03:31:50,272 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:31:50,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33222, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:31:50,292 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=8394b182524b,41967,1731468706868 2024-11-13T03:31:50,293 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:33705/hbase 2024-11-13T03:31:50,306 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=366, OpenFileDescriptor=611, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=3236 2024-11-13T03:31:50,325 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:50,327 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:50,328 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:50,332 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-27045429, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-27045429, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:50,348 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-27045429/hregion-27045429.1731468710334, exclude list is [], retry=0 2024-11-13T03:31:50,352 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:50,352 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:50,352 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:50,356 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-27045429/hregion-27045429.1731468710334 2024-11-13T03:31:50,357 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:50,357 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => a2b076d85271be8690eb0a743f01b4ae, NAME => 'testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:31:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:31:50,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:31:50,371 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741838_1014 (size=64) 2024-11-13T03:31:50,372 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:50,374 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,377 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2b076d85271be8690eb0a743f01b4ae columnFamilyName a 2024-11-13T03:31:50,377 DEBUG [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:50,377 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] regionserver.HStore(327): Store=a2b076d85271be8690eb0a743f01b4ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:50,378 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,379 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,380 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,381 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,381 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,383 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,387 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:50,388 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a2b076d85271be8690eb0a743f01b4ae; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62243552, jitterRate=-0.07249879837036133}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:31:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a2b076d85271be8690eb0a743f01b4ae: Writing region info on filesystem at 1731468710372Initializing all the Stores at 1731468710374 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468710374Cleaning up temporary data from old regions at 1731468710381 (+7 ms)Region opened successfully at 1731468710389 (+8 ms) 2024-11-13T03:31:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing a2b076d85271be8690eb0a743f01b4ae, disabling compactions & flushes 2024-11-13T03:31:50,389 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. 2024-11-13T03:31:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. 2024-11-13T03:31:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. after waiting 0 ms 2024-11-13T03:31:50,389 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. 2024-11-13T03:31:50,390 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. 2024-11-13T03:31:50,390 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for a2b076d85271be8690eb0a743f01b4ae: Waiting for close lock at 1731468710389Disabling compacts and flushes for region at 1731468710389Disabling writes for close at 1731468710389Writing region close event to WAL at 1731468710389Closed at 1731468710389 2024-11-13T03:31:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741837_1013 (size=93) 2024-11-13T03:31:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741837_1013 (size=93) 2024-11-13T03:31:50,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741837_1013 (size=93) 2024-11-13T03:31:50,400 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:50,401 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-27045429:(num 1731468710334) 2024-11-13T03:31:50,402 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:31:50,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741839_1015 (size=369) 2024-11-13T03:31:50,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741839_1015 (size=369) 2024-11-13T03:31:50,419 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741839_1015 (size=369) 2024-11-13T03:31:50,420 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:31:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741840_1016 (size=231) 2024-11-13T03:31:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741840_1016 (size=231) 2024-11-13T03:31:50,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741840_1016 (size=231) 2024-11-13T03:31:50,454 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1, size=369 (369bytes) 2024-11-13T03:31:50,454 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-11-13T03:31:50,455 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-11-13T03:31:50,455 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1 2024-11-13T03:31:50,460 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1 after 3ms 2024-11-13T03:31:50,467 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1 took 14ms 2024-11-13T03:31:50,471 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1 so closing down 2024-11-13T03:31:50,474 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T03:31:50,476 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T03:31:50,476 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:50,477 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:50,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741841_1017 (size=369) 2024-11-13T03:31:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741841_1017 (size=369) 2024-11-13T03:31:50,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741841_1017 (size=369) 2024-11-13T03:31:50,489 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:31:50,492 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002 2024-11-13T03:31:50,497 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T03:31:50,497 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1, journal: Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1, size=369 (369bytes) at 1731468710454Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1 so closing down at 1731468710471 (+17 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp at 1731468710476 (+5 ms)3 split writer threads finished at 1731468710477 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468710489 (+12 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002 at 1731468710492 (+3 ms)Processed 2 edits across 1 Regions in 25 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731468710497 (+5 ms) 2024-11-13T03:31:50,512 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2, size=231 (231bytes) 2024-11-13T03:31:50,512 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2 2024-11-13T03:31:50,513 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2 after 1ms 2024-11-13T03:31:50,517 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2 took 5ms 2024-11-13T03:31:50,520 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2 so closing down 2024-11-13T03:31:50,520 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:50,523 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T03:31:50,525 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T03:31:50,525 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:50,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741842_1018 (size=231) 2024-11-13T03:31:50,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741842_1018 (size=231) 2024-11-13T03:31:50,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741842_1018 (size=231) 2024-11-13T03:31:50,534 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:31:50,540 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002-wal-2.temp, length=231 2024-11-13T03:31:50,541 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T03:31:50,542 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2, journal: Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2, size=231 (231bytes) at 1731468710512Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2 so closing down at 1731468710520 (+8 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002-wal-2.temp at 1731468710525 (+5 ms)3 split writer threads finished at 1731468710525Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468710534 (+9 ms)Processed 1 edits across 1 Regions in 23 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731468710542 (+8 ms) 2024-11-13T03:31:50,542 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:50,544 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:50,557 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal.1731468710545, exclude list is [], retry=0 2024-11-13T03:31:50,562 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:50,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:50,563 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:50,566 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal.1731468710545 2024-11-13T03:31:50,566 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:31:50,567 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => a2b076d85271be8690eb0a743f01b4ae, NAME => 'testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:50,567 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:50,567 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,567 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,569 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,570 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a2b076d85271be8690eb0a743f01b4ae columnFamilyName a 2024-11-13T03:31:50,571 DEBUG [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:50,572 INFO [StoreOpener-a2b076d85271be8690eb0a743f01b4ae-1 {}] regionserver.HStore(327): Store=a2b076d85271be8690eb0a743f01b4ae/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:50,572 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,573 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,576 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,577 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002 2024-11-13T03:31:50,586 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002 2024-11-13T03:31:50,589 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing a2b076d85271be8690eb0a743f01b4ae 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T03:31:50,643 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/.tmp/a/175015ee687a4f07be3888f76e22c8bb is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731468710401/Put/seqid=0 2024-11-13T03:31:50,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:31:50,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:31:50,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741844_1020 (size=5170) 2024-11-13T03:31:50,657 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/.tmp/a/175015ee687a4f07be3888f76e22c8bb 2024-11-13T03:31:50,699 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/.tmp/a/175015ee687a4f07be3888f76e22c8bb as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/a/175015ee687a4f07be3888f76e22c8bb 2024-11-13T03:31:50,708 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/a/175015ee687a4f07be3888f76e22c8bb, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T03:31:50,714 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a2b076d85271be8690eb0a743f01b4ae in 123ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T03:31:50,716 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/0000000000000000002 2024-11-13T03:31:50,717 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,717 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,720 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for a2b076d85271be8690eb0a743f01b4ae 2024-11-13T03:31:50,724 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/a2b076d85271be8690eb0a743f01b4ae/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T03:31:50,725 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened a2b076d85271be8690eb0a743f01b4ae; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70628514, jitterRate=0.0524468719959259}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:31:50,726 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for a2b076d85271be8690eb0a743f01b4ae: Writing region info on filesystem at 1731468710567Initializing all the Stores at 1731468710569 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468710569Obtaining lock to block concurrent updates at 1731468710589 (+20 ms)Preparing flush snapshotting stores in a2b076d85271be8690eb0a743f01b4ae at 1731468710589Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731468710591 (+2 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731468710325.a2b076d85271be8690eb0a743f01b4ae. at 1731468710591Flushing a2b076d85271be8690eb0a743f01b4ae/a: creating writer at 1731468710593 (+2 ms)Flushing a2b076d85271be8690eb0a743f01b4ae/a: appending metadata at 1731468710628 (+35 ms)Flushing a2b076d85271be8690eb0a743f01b4ae/a: closing flushed file at 1731468710632 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@d2cd1fc: reopening flushed file at 1731468710697 (+65 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for a2b076d85271be8690eb0a743f01b4ae in 123ms, sequenceid=2, compaction requested=false; wal=null at 1731468710714 (+17 ms)Cleaning up temporary data from old regions at 1731468710717 (+3 ms)Region opened successfully at 1731468710726 (+9 ms) 2024-11-13T03:31:50,754 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit0 Thread=377 (was 366) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40930 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:56694 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:42730 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:42628 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40774 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:33705/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=703 (was 611) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=3207 (was 3236) 2024-11-13T03:31:50,764 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=377, OpenFileDescriptor=703, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=3206 2024-11-13T03:31:50,779 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:50,781 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:50,782 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:50,785 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-47359955, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-47359955, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:50,797 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-47359955/hregion-47359955.1731468710786, exclude list is [], retry=0 2024-11-13T03:31:50,801 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:50,801 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:50,801 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:50,805 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-47359955/hregion-47359955.1731468710786 2024-11-13T03:31:50,808 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:31:50,808 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 3d4bd689e927c1d93e9ff2de0e1d9434, NAME => 'testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:31:50,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:31:50,820 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:31:50,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741846_1022 (size=64) 2024-11-13T03:31:50,821 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:50,823 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,826 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d4bd689e927c1d93e9ff2de0e1d9434 columnFamilyName a 2024-11-13T03:31:50,826 DEBUG [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:50,827 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] regionserver.HStore(327): Store=3d4bd689e927c1d93e9ff2de0e1d9434/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:50,827 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,828 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,828 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,829 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,829 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,831 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,834 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:50,834 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3d4bd689e927c1d93e9ff2de0e1d9434; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59053900, jitterRate=-0.12002831697463989}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3d4bd689e927c1d93e9ff2de0e1d9434: Writing region info on filesystem at 1731468710821Initializing all the Stores at 1731468710823 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468710823Cleaning up temporary data from old regions at 1731468710829 (+6 ms)Region opened successfully at 1731468710834 (+5 ms) 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 3d4bd689e927c1d93e9ff2de0e1d9434, disabling compactions & flushes 2024-11-13T03:31:50,835 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. after waiting 0 ms 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. 2024-11-13T03:31:50,835 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. 2024-11-13T03:31:50,835 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 3d4bd689e927c1d93e9ff2de0e1d9434: Waiting for close lock at 1731468710835Disabling compacts and flushes for region at 1731468710835Disabling writes for close at 1731468710835Writing region close event to WAL at 1731468710835Closed at 1731468710835 2024-11-13T03:31:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741845_1021 (size=93) 2024-11-13T03:31:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741845_1021 (size=93) 2024-11-13T03:31:50,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741845_1021 (size=93) 2024-11-13T03:31:50,845 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:50,845 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-47359955:(num 1731468710786) 2024-11-13T03:31:50,846 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:31:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741847_1023 (size=369) 2024-11-13T03:31:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741847_1023 (size=369) 2024-11-13T03:31:50,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741847_1023 (size=369) 2024-11-13T03:31:50,861 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-11-13T03:31:50,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741848_1024 (size=231) 2024-11-13T03:31:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741848_1024 (size=231) 2024-11-13T03:31:50,872 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741848_1024 (size=231) 2024-11-13T03:31:50,892 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2, size=231 (231bytes) 2024-11-13T03:31:50,892 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2 2024-11-13T03:31:50,893 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2 after 1ms 2024-11-13T03:31:50,897 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2 took 6ms 2024-11-13T03:31:50,899 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2 so closing down 2024-11-13T03:31:50,900 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:50,902 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-11-13T03:31:50,903 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp 2024-11-13T03:31:50,903 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:50,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741849_1025 (size=231) 2024-11-13T03:31:50,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741849_1025 (size=231) 2024-11-13T03:31:50,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741849_1025 (size=231) 2024-11-13T03:31:50,912 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:31:50,914 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 2024-11-13T03:31:50,914 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2, size=231, length=231, corrupted=false, cancelled=false 2024-11-13T03:31:50,914 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2, journal: Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2, size=231 (231bytes) at 1731468710892Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2 so closing down at 1731468710899 (+7 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp at 1731468710903 (+4 ms)3 split writer threads finished at 1731468710903Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468710912 (+9 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 at 1731468710914 (+2 ms)Processed 1 edits across 1 Regions in 17 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-2, size=231, length=231, corrupted=false, cancelled=false at 1731468710914 2024-11-13T03:31:50,935 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1, size=369 (369bytes) 2024-11-13T03:31:50,935 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1 2024-11-13T03:31:50,936 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1 after 1ms 2024-11-13T03:31:50,941 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1 took 5ms 2024-11-13T03:31:50,944 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1 so closing down 2024-11-13T03:31:50,944 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:50,946 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-11-13T03:31:50,948 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp 2024-11-13T03:31:50,948 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741850_1026 (size=369) 2024-11-13T03:31:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741850_1026 (size=369) 2024-11-13T03:31:50,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741850_1026 (size=369) 2024-11-13T03:31:50,958 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:31:50,963 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002, length=231 2024-11-13T03:31:50,965 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 2024-11-13T03:31:50,965 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1, size=369, length=369, corrupted=false, cancelled=false 2024-11-13T03:31:50,965 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1, journal: Splitting hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1, size=369 (369bytes) at 1731468710935Finishing writing output for hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1 so closing down at 1731468710944 (+9 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp at 1731468710948 (+4 ms)3 split writer threads finished at 1731468710948Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468710958 (+10 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 at 1731468710965 (+7 ms)Processed 2 edits across 1 Regions in 24 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal-1, size=369, length=369, corrupted=false, cancelled=false at 1731468710965 2024-11-13T03:31:50,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:50,968 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:50,981 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal.1731468710968, exclude list is [], retry=0 2024-11-13T03:31:50,984 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:50,985 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:50,985 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:50,988 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal.1731468710968 2024-11-13T03:31:50,988 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:31:50,988 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3d4bd689e927c1d93e9ff2de0e1d9434, NAME => 'testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:50,988 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:50,989 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,989 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,990 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,992 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3d4bd689e927c1d93e9ff2de0e1d9434 columnFamilyName a 2024-11-13T03:31:50,993 DEBUG [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:50,993 INFO [StoreOpener-3d4bd689e927c1d93e9ff2de0e1d9434-1 {}] regionserver.HStore(327): Store=3d4bd689e927c1d93e9ff2de0e1d9434/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:50,994 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,995 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,997 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:50,998 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 2024-11-13T03:31:51,003 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 2024-11-13T03:31:51,003 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3d4bd689e927c1d93e9ff2de0e1d9434 1/1 column families, dataSize=108 B heapSize=512 B 2024-11-13T03:31:51,020 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/.tmp/a/8881b2ac77de459da9420a2299895906 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1731468710845/Put/seqid=0 2024-11-13T03:31:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:31:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:31:51,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741852_1028 (size=5170) 2024-11-13T03:31:51,029 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/.tmp/a/8881b2ac77de459da9420a2299895906 2024-11-13T03:31:51,039 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/.tmp/a/8881b2ac77de459da9420a2299895906 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/a/8881b2ac77de459da9420a2299895906 2024-11-13T03:31:51,049 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/a/8881b2ac77de459da9420a2299895906, entries=2, sequenceid=2, filesize=5.0 K 2024-11-13T03:31:51,049 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3d4bd689e927c1d93e9ff2de0e1d9434 in 46ms, sequenceid=2, compaction requested=false; wal=null 2024-11-13T03:31:51,050 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/0000000000000000002 2024-11-13T03:31:51,051 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:51,051 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:51,053 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3d4bd689e927c1d93e9ff2de0e1d9434 2024-11-13T03:31:51,056 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/3d4bd689e927c1d93e9ff2de0e1d9434/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-11-13T03:31:51,058 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3d4bd689e927c1d93e9ff2de0e1d9434; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67141993, jitterRate=4.93660569190979E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:31:51,058 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3d4bd689e927c1d93e9ff2de0e1d9434: Writing region info on filesystem at 1731468710989Initializing all the Stores at 1731468710990 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468710990Obtaining lock to block concurrent updates at 1731468711003 (+13 ms)Preparing flush snapshotting stores in 3d4bd689e927c1d93e9ff2de0e1d9434 at 1731468711003Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1731468711004 (+1 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1731468710779.3d4bd689e927c1d93e9ff2de0e1d9434. at 1731468711004Flushing 3d4bd689e927c1d93e9ff2de0e1d9434/a: creating writer at 1731468711004Flushing 3d4bd689e927c1d93e9ff2de0e1d9434/a: appending metadata at 1731468711018 (+14 ms)Flushing 3d4bd689e927c1d93e9ff2de0e1d9434/a: closing flushed file at 1731468711019 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50dceba4: reopening flushed file at 1731468711037 (+18 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for 3d4bd689e927c1d93e9ff2de0e1d9434 in 46ms, sequenceid=2, compaction requested=false; wal=null at 1731468711049 (+12 ms)Cleaning up temporary data from old regions at 1731468711051 (+2 ms)Region opened successfully at 1731468711058 (+7 ms) 2024-11-13T03:31:51,074 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testNameConflictWhenSplit1 Thread=387 (was 377) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:42628 [Waiting for operation #16] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:56754 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:41000 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40774 [Waiting for operation #19] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:42816 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=785 (was 703) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=3203 (was 3206) 2024-11-13T03:31:51,084 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=387, OpenFileDescriptor=785, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=3203 2024-11-13T03:31:51,103 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:51,105 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:51,106 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:51,109 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85833336, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-85833336, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:51,122 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85833336/hregion-85833336.1731468711110, exclude list is [], retry=0 2024-11-13T03:31:51,126 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:51,127 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:51,127 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:51,129 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85833336/hregion-85833336.1731468711110 2024-11-13T03:31:51,129 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:51,129 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 22fedca66885b803e3faacb1928021ca, NAME => 'testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:31:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:31:51,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:31:51,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741854_1030 (size=64) 2024-11-13T03:31:51,140 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:51,142 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,144 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName a 2024-11-13T03:31:51,144 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:51,145 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:51,145 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,147 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName b 2024-11-13T03:31:51,147 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:51,148 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:51,148 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,151 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName c 2024-11-13T03:31:51,151 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:51,152 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:51,152 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,153 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,154 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,155 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,156 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,156 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:31:51,158 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:51,160 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:51,161 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22fedca66885b803e3faacb1928021ca; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59732114, jitterRate=-0.1099221408367157}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:31:51,161 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22fedca66885b803e3faacb1928021ca: Writing region info on filesystem at 1731468711141Initializing all the Stores at 1731468711142 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468711142Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468711142Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468711142Cleaning up temporary data from old regions at 1731468711156 (+14 ms)Region opened successfully at 1731468711161 (+5 ms) 2024-11-13T03:31:51,161 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22fedca66885b803e3faacb1928021ca, disabling compactions & flushes 2024-11-13T03:31:51,161 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:51,161 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:51,161 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. after waiting 0 ms 2024-11-13T03:31:51,161 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:51,162 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:51,162 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22fedca66885b803e3faacb1928021ca: Waiting for close lock at 1731468711161Disabling compacts and flushes for region at 1731468711161Disabling writes for close at 1731468711161Writing region close event to WAL at 1731468711162 (+1 ms)Closed at 1731468711162 2024-11-13T03:31:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741853_1029 (size=93) 2024-11-13T03:31:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741853_1029 (size=93) 2024-11-13T03:31:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741853_1029 (size=93) 2024-11-13T03:31:51,168 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:51,168 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85833336:(num 1731468711110) 2024-11-13T03:31:51,168 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:51,171 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:51,187 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, exclude list is [], retry=0 2024-11-13T03:31:51,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:51,190 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:51,191 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:51,193 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 2024-11-13T03:31:51,193 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:51,418 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, size=0 (0bytes) 2024-11-13T03:31:51,419 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 might be still open, length is 0 2024-11-13T03:31:51,419 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 2024-11-13T03:31:51,420 WARN [IPC Server handler 1 on default port 33705 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-11-13T03:31:51,420 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 after 1ms 2024-11-13T03:31:51,956 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:56794 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:56794 dst: /127.0.0.1:43451 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43451 remote=/127.0.0.1:56794]. Total timeout mills is 60000, 59430 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:51,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:42852 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:37801:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:42852 dst: /127.0.0.1:37801 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:51,957 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:41034 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:43285:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:41034 dst: /127.0.0.1:43285 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:51,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741855_1032 (size=470675) 2024-11-13T03:31:51,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741855_1032 (size=470675) 2024-11-13T03:31:51,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741855_1032 (size=470675) 2024-11-13T03:31:55,380 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:31:55,421 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 after 4002ms 2024-11-13T03:31:55,427 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 took 4009ms 2024-11-13T03:31:55,431 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731468711172.temp 2024-11-13T03:31:55,433 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp 2024-11-13T03:31:55,439 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T03:31:55,508 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172; continuing. 2024-11-13T03:31:55,508 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 so closing down 2024-11-13T03:31:55,508 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:55,509 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:55,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741856_1033 (size=470683) 2024-11-13T03:31:55,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741856_1033 (size=470683) 2024-11-13T03:31:55,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741856_1033 (size=470683) 2024-11-13T03:31:55,512 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp (wrote 3002 edits, skipped 0 edits in 28 ms) 2024-11-13T03:31:55,514 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 2024-11-13T03:31:55,514 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:31:55,514 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, journal: Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, size=0 (0bytes) at 1731468711419Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp at 1731468715434 (+4015 ms)Split 1024 edits, skipped 0 edits. at 1731468715483 (+49 ms)Split 2048 edits, skipped 0 edits. at 1731468715497 (+14 ms)Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 so closing down at 1731468715508 (+11 ms)3 split writer threads finished at 1731468715509 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp (wrote 3002 edits, skipped 0 edits in 28 ms) at 1731468715512 (+3 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000000001-wal.1731468711172.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 at 1731468715514 (+2 ms)Processed 3002 edits across 1 Regions in 87 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172, size=0, length=0, corrupted=false, cancelled=false at 1731468715514 2024-11-13T03:31:55,516 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468711172 2024-11-13T03:31:55,518 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 2024-11-13T03:31:55,518 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:55,520 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:55,534 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468715521, exclude list is [], retry=0 2024-11-13T03:31:55,537 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:55,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:55,538 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:55,540 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468715521 2024-11-13T03:31:55,541 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:55,541 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:55,543 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,545 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName a 2024-11-13T03:31:55,545 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:55,545 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:55,545 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,547 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName b 2024-11-13T03:31:55,547 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:55,547 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:55,547 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,548 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 22fedca66885b803e3faacb1928021ca columnFamilyName c 2024-11-13T03:31:55,548 DEBUG [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:55,549 INFO [StoreOpener-22fedca66885b803e3faacb1928021ca-1 {}] regionserver.HStore(327): Store=22fedca66885b803e3faacb1928021ca/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:55,549 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,550 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,552 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:55,553 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 2024-11-13T03:31:55,595 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:31:55,949 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22fedca66885b803e3faacb1928021ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:31:55,987 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/5a552bd2b41b4b95ba78df06c38377ac is 62, key is testReplayEditsWrittenIntoWAL/a:100/1731468711205/Put/seqid=0 2024-11-13T03:31:55,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:31:55,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:31:55,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741858_1035 (size=50463) 2024-11-13T03:31:55,996 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/5a552bd2b41b4b95ba78df06c38377ac 2024-11-13T03:31:56,003 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/5a552bd2b41b4b95ba78df06c38377ac as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/a/5a552bd2b41b4b95ba78df06c38377ac 2024-11-13T03:31:56,010 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/a/5a552bd2b41b4b95ba78df06c38377ac, entries=754, sequenceid=754, filesize=49.3 K 2024-11-13T03:31:56,011 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 22fedca66885b803e3faacb1928021ca in 61ms, sequenceid=754, compaction requested=false; wal=null 2024-11-13T03:31:56,027 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:31:56,028 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22fedca66885b803e3faacb1928021ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:31:56,036 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/27e933ed484248b1a26fa96443ecdc56 is 62, key is testReplayEditsWrittenIntoWAL/a:754/1731468711258/Put/seqid=0 2024-11-13T03:31:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:31:56,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:31:56,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741859_1036 (size=20072) 2024-11-13T03:31:56,047 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/27e933ed484248b1a26fa96443ecdc56 2024-11-13T03:31:56,072 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/d28a57aaa5774a319457fabe012f8099 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1731468711282/Put/seqid=0 2024-11-13T03:31:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:31:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:31:56,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741860_1037 (size=35835) 2024-11-13T03:31:56,081 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/d28a57aaa5774a319457fabe012f8099 2024-11-13T03:31:56,087 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/a/27e933ed484248b1a26fa96443ecdc56 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/a/27e933ed484248b1a26fa96443ecdc56 2024-11-13T03:31:56,095 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/a/27e933ed484248b1a26fa96443ecdc56, entries=246, sequenceid=1508, filesize=19.6 K 2024-11-13T03:31:56,096 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/d28a57aaa5774a319457fabe012f8099 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/b/d28a57aaa5774a319457fabe012f8099 2024-11-13T03:31:56,103 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/b/d28a57aaa5774a319457fabe012f8099, entries=508, sequenceid=1508, filesize=35.0 K 2024-11-13T03:31:56,103 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 22fedca66885b803e3faacb1928021ca in 75ms, sequenceid=1508, compaction requested=false; wal=null 2024-11-13T03:31:56,118 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:31:56,119 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22fedca66885b803e3faacb1928021ca 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-11-13T03:31:56,126 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/606390e549654dbe806ec14c0f21eb97 is 62, key is testReplayEditsWrittenIntoWAL/b:508/1731468711306/Put/seqid=0 2024-11-13T03:31:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:31:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:31:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741861_1038 (size=35082) 2024-11-13T03:31:56,135 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/606390e549654dbe806ec14c0f21eb97 2024-11-13T03:31:56,158 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/055e42472c9f4d12bd66aa1e81e6da6e is 62, key is testReplayEditsWrittenIntoWAL/c:100/1731468711340/Put/seqid=0 2024-11-13T03:31:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:31:56,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:31:56,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741862_1039 (size=20825) 2024-11-13T03:31:56,166 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/055e42472c9f4d12bd66aa1e81e6da6e 2024-11-13T03:31:56,174 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/b/606390e549654dbe806ec14c0f21eb97 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/b/606390e549654dbe806ec14c0f21eb97 2024-11-13T03:31:56,182 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/b/606390e549654dbe806ec14c0f21eb97, entries=492, sequenceid=2262, filesize=34.3 K 2024-11-13T03:31:56,183 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/055e42472c9f4d12bd66aa1e81e6da6e as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/c/055e42472c9f4d12bd66aa1e81e6da6e 2024-11-13T03:31:56,189 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/c/055e42472c9f4d12bd66aa1e81e6da6e, entries=262, sequenceid=2262, filesize=20.3 K 2024-11-13T03:31:56,189 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 22fedca66885b803e3faacb1928021ca in 70ms, sequenceid=2262, compaction requested=false; wal=null 2024-11-13T03:31:56,196 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1731468711385/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:56,199 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 2024-11-13T03:31:56,199 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-11-13T03:31:56,199 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 22fedca66885b803e3faacb1928021ca 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-11-13T03:31:56,206 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/499ed552bc6546a9ba7af3bc654a72b3 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1731468711346/Put/seqid=0 2024-11-13T03:31:56,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:31:56,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:31:56,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741863_1040 (size=50301) 2024-11-13T03:31:56,215 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/499ed552bc6546a9ba7af3bc654a72b3 2024-11-13T03:31:56,221 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 499ed552bc6546a9ba7af3bc654a72b3 2024-11-13T03:31:56,223 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/.tmp/c/499ed552bc6546a9ba7af3bc654a72b3 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/c/499ed552bc6546a9ba7af3bc654a72b3 2024-11-13T03:31:56,230 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 499ed552bc6546a9ba7af3bc654a72b3 2024-11-13T03:31:56,230 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/c/499ed552bc6546a9ba7af3bc654a72b3, entries=739, sequenceid=3002, filesize=49.1 K 2024-11-13T03:31:56,231 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 22fedca66885b803e3faacb1928021ca in 32ms, sequenceid=3002, compaction requested=false; wal=null 2024-11-13T03:31:56,232 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/0000000000000003002 2024-11-13T03:31:56,233 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:56,234 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:56,234 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-13T03:31:56,236 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 22fedca66885b803e3faacb1928021ca 2024-11-13T03:31:56,238 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenIntoWAL/22fedca66885b803e3faacb1928021ca/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-11-13T03:31:56,240 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 22fedca66885b803e3faacb1928021ca; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69569773, jitterRate=0.03667040169239044}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-13T03:31:56,240 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 22fedca66885b803e3faacb1928021ca: Writing region info on filesystem at 1731468715541Initializing all the Stores at 1731468715543 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468715543Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468715543Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468715543Cleaning up temporary data from old regions at 1731468716234 (+691 ms)Region opened successfully at 1731468716240 (+6 ms) 2024-11-13T03:31:56,308 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 22fedca66885b803e3faacb1928021ca, disabling compactions & flushes 2024-11-13T03:31:56,308 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:56,308 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:56,308 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. after waiting 0 ms 2024-11-13T03:31:56,308 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:56,310 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1731468711104.22fedca66885b803e3faacb1928021ca. 2024-11-13T03:31:56,310 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 22fedca66885b803e3faacb1928021ca: Waiting for close lock at 1731468716308Disabling compacts and flushes for region at 1731468716308Disabling writes for close at 1731468716308Writing region close event to WAL at 1731468716310 (+2 ms)Closed at 1731468716310 2024-11-13T03:31:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741857_1034 (size=93) 2024-11-13T03:31:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741857_1034 (size=93) 2024-11-13T03:31:56,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741857_1034 (size=93) 2024-11-13T03:31:56,317 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:56,317 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468715521) 2024-11-13T03:31:56,332 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=401 (was 387) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-867200125_22 at /127.0.0.1:40182 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-867200125_22 at /127.0.0.1:47222 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:45567 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:41461 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:45567 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:41461 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:33705 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33705 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-867200125_22 at /127.0.0.1:47230 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-867200125_22 at /127.0.0.1:38080 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=865 (was 785) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 364), ProcessCount=11 (was 11), AvailableMemoryMB=3150 (was 3203) 2024-11-13T03:31:56,342 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#test2727 Thread=401, OpenFileDescriptor=865, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=3149 2024-11-13T03:31:56,364 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:56,367 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:56,369 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:56,371 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67281967, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-67281967, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:56,385 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67281967/hregion-67281967.1731468716372, exclude list is [], retry=0 2024-11-13T03:31:56,388 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:56,389 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:56,389 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:56,394 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67281967/hregion-67281967.1731468716372 2024-11-13T03:31:56,395 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:56,395 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 7869ac0070eeb7f4c6fae43c7d0818f2, NAME => 'test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:31:56,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:31:56,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:31:56,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741865_1042 (size=43) 2024-11-13T03:31:56,408 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:56,409 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,411 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName a 2024-11-13T03:31:56,411 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:56,411 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:56,412 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,413 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName b 2024-11-13T03:31:56,413 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:56,414 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:56,414 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,416 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName c 2024-11-13T03:31:56,416 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:56,417 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:56,417 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,418 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,419 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,420 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,420 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,421 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:31:56,422 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:56,425 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:56,425 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7869ac0070eeb7f4c6fae43c7d0818f2; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72112186, jitterRate=0.07455530762672424}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:31:56,426 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7869ac0070eeb7f4c6fae43c7d0818f2: Writing region info on filesystem at 1731468716408Initializing all the Stores at 1731468716409 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468716409Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468716409Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468716409Cleaning up temporary data from old regions at 1731468716420 (+11 ms)Region opened successfully at 1731468716426 (+6 ms) 2024-11-13T03:31:56,426 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7869ac0070eeb7f4c6fae43c7d0818f2, disabling compactions & flushes 2024-11-13T03:31:56,427 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:56,427 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:56,427 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. after waiting 0 ms 2024-11-13T03:31:56,427 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:56,427 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:56,427 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7869ac0070eeb7f4c6fae43c7d0818f2: Waiting for close lock at 1731468716426Disabling compacts and flushes for region at 1731468716426Disabling writes for close at 1731468716427 (+1 ms)Writing region close event to WAL at 1731468716427Closed at 1731468716427 2024-11-13T03:31:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741864_1041 (size=93) 2024-11-13T03:31:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741864_1041 (size=93) 2024-11-13T03:31:56,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741864_1041 (size=93) 2024-11-13T03:31:56,439 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:56,439 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67281967:(num 1731468716372) 2024-11-13T03:31:56,439 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:56,442 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:56,455 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, exclude list is [], retry=0 2024-11-13T03:31:56,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:56,458 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:56,459 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:56,461 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 2024-11-13T03:31:56,461 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:56,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741866_1043 (size=344306) 2024-11-13T03:31:56,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741866_1043 (size=344306) 2024-11-13T03:31:56,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741866_1043 (size=344306) 2024-11-13T03:31:56,627 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, size=336.2 K (344306bytes) 2024-11-13T03:31:56,627 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 2024-11-13T03:31:56,627 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 after 0ms 2024-11-13T03:31:56,632 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 took 5ms 2024-11-13T03:31:56,636 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1731468716442.temp 2024-11-13T03:31:56,638 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp 2024-11-13T03:31:56,670 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 so closing down 2024-11-13T03:31:56,670 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:56,670 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:56,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741867_1044 (size=344306) 2024-11-13T03:31:56,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741867_1044 (size=344306) 2024-11-13T03:31:56,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741867_1044 (size=344306) 2024-11-13T03:31:56,674 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp (wrote 3000 edits, skipped 0 edits in 11 ms) 2024-11-13T03:31:56,676 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp to hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 2024-11-13T03:31:56,676 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, size=336.2 K, length=344306, corrupted=false, cancelled=false 2024-11-13T03:31:56,676 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, journal: Splitting hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, size=336.2 K (344306bytes) at 1731468716627Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp at 1731468716638 (+11 ms)Split 1024 edits, skipped 0 edits. at 1731468716647 (+9 ms)Split 2048 edits, skipped 0 edits. at 1731468716659 (+12 ms)Finishing writing output for hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 so closing down at 1731468716670 (+11 ms)3 split writer threads finished at 1731468716670Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp (wrote 3000 edits, skipped 0 edits in 11 ms) at 1731468716674 (+4 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000000001-wal.1731468716442.temp to hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 at 1731468716676 (+2 ms)Processed 3000 edits across 1 Regions in 44 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442, size=336.2 K, length=344306, corrupted=false, cancelled=false at 1731468716676 2024-11-13T03:31:56,678 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716442 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468716442 2024-11-13T03:31:56,679 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 2024-11-13T03:31:56,680 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:56,682 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:56,694 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, exclude list is [], retry=0 2024-11-13T03:31:56,697 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:56,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:56,698 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:56,699 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 2024-11-13T03:31:56,700 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:31:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741868_1045 (size=344433) 2024-11-13T03:31:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741868_1045 (size=344433) 2024-11-13T03:31:56,915 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741868_1045 (size=344433) 2024-11-13T03:31:56,934 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, size=336.4 K (344433bytes) 2024-11-13T03:31:56,935 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 2024-11-13T03:31:56,935 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 after 0ms 2024-11-13T03:31:56,940 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 took 6ms 2024-11-13T03:31:56,944 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1731468716682.temp 2024-11-13T03:31:56,946 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp 2024-11-13T03:31:56,995 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 so closing down 2024-11-13T03:31:56,995 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:56,995 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:57,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741869_1046 (size=344433) 2024-11-13T03:31:57,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741869_1046 (size=344433) 2024-11-13T03:31:57,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741869_1046 (size=344433) 2024-11-13T03:31:57,010 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp (wrote 3000 edits, skipped 0 edits in 37 ms) 2024-11-13T03:31:57,012 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp to hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 2024-11-13T03:31:57,012 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, size=336.4 K, length=344433, corrupted=false, cancelled=false 2024-11-13T03:31:57,012 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, journal: Splitting hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, size=336.4 K (344433bytes) at 1731468716935Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp at 1731468716946 (+11 ms)Split 1024 edits, skipped 0 edits. at 1731468716980 (+34 ms)Split 2048 edits, skipped 0 edits. at 1731468716986 (+6 ms)Finishing writing output for hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 so closing down at 1731468716995 (+9 ms)3 split writer threads finished at 1731468716995Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp (wrote 3000 edits, skipped 0 edits in 37 ms) at 1731468717010 (+15 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003001-wal.1731468716682.temp to hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 at 1731468717012 (+2 ms)Processed 3000 edits across 1 Regions in 72 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682, size=336.4 K, length=344433, corrupted=false, cancelled=false at 1731468717012 2024-11-13T03:31:57,021 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468716682 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468716682 2024-11-13T03:31:57,022 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 2024-11-13T03:31:57,023 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:57,026 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/test2727-manual,16010,1731468716363, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:57,049 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468717026, exclude list is [], retry=0 2024-11-13T03:31:57,052 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:57,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:57,053 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:57,055 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1731468716363/wal.1731468717026 2024-11-13T03:31:57,056 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:31:57,056 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 7869ac0070eeb7f4c6fae43c7d0818f2, NAME => 'test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:57,056 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:57,056 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,057 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,059 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,060 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName a 2024-11-13T03:31:57,060 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,061 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,061 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,062 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName b 2024-11-13T03:31:57,062 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,063 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,063 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,064 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7869ac0070eeb7f4c6fae43c7d0818f2 columnFamilyName c 2024-11-13T03:31:57,064 DEBUG [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,065 INFO [StoreOpener-7869ac0070eeb7f4c6fae43c7d0818f2-1 {}] regionserver.HStore(327): Store=7869ac0070eeb7f4c6fae43c7d0818f2/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,065 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,069 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,070 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 2024-11-13T03:31:57,098 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 2024-11-13T03:31:57,099 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 2024-11-13T03:31:57,128 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 2024-11-13T03:31:57,128 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 7869ac0070eeb7f4c6fae43c7d0818f2 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-11-13T03:31:57,155 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/a/165629e4bfd3454688b88bb162362a79 is 41, key is test2727/a:100/1731468716708/Put/seqid=0 2024-11-13T03:31:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:31:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:31:57,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741871_1048 (size=84227) 2024-11-13T03:31:57,163 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/a/165629e4bfd3454688b88bb162362a79 2024-11-13T03:31:57,190 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/b/c2eb0ca541bb4a37be9968bd26a218d8 is 41, key is test2727/b:100/1731468716781/Put/seqid=0 2024-11-13T03:31:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:31:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:31:57,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741872_1049 (size=84609) 2024-11-13T03:31:57,196 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/b/c2eb0ca541bb4a37be9968bd26a218d8 2024-11-13T03:31:57,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T03:31:57,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-13T03:31:57,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:31:57,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-13T03:31:57,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-13T03:31:57,216 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-13T03:31:57,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-11-13T03:31:57,217 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-11-13T03:31:57,229 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/c/f3e6b842c789485f90d98d786d4e2d81 is 41, key is test2727/c:100/1731468716853/Put/seqid=0 2024-11-13T03:31:57,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:31:57,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:31:57,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741873_1050 (size=84609) 2024-11-13T03:31:57,235 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/c/f3e6b842c789485f90d98d786d4e2d81 2024-11-13T03:31:57,242 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/a/165629e4bfd3454688b88bb162362a79 as hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/a/165629e4bfd3454688b88bb162362a79 2024-11-13T03:31:57,249 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/a/165629e4bfd3454688b88bb162362a79, entries=2000, sequenceid=6000, filesize=82.3 K 2024-11-13T03:31:57,250 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/b/c2eb0ca541bb4a37be9968bd26a218d8 as hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/b/c2eb0ca541bb4a37be9968bd26a218d8 2024-11-13T03:31:57,257 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/b/c2eb0ca541bb4a37be9968bd26a218d8, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T03:31:57,258 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/.tmp/c/f3e6b842c789485f90d98d786d4e2d81 as hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/c/f3e6b842c789485f90d98d786d4e2d81 2024-11-13T03:31:57,265 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/c/f3e6b842c789485f90d98d786d4e2d81, entries=2000, sequenceid=6000, filesize=82.6 K 2024-11-13T03:31:57,265 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 7869ac0070eeb7f4c6fae43c7d0818f2 in 137ms, sequenceid=6000, compaction requested=false; wal=null 2024-11-13T03:31:57,266 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000003000 2024-11-13T03:31:57,267 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/0000000000000006000 2024-11-13T03:31:57,268 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,268 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,269 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:31:57,270 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 7869ac0070eeb7f4c6fae43c7d0818f2 2024-11-13T03:31:57,273 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/test2727/7869ac0070eeb7f4c6fae43c7d0818f2/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-11-13T03:31:57,274 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 7869ac0070eeb7f4c6fae43c7d0818f2; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65794577, jitterRate=-0.019584402441978455}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:31:57,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 7869ac0070eeb7f4c6fae43c7d0818f2: Writing region info on filesystem at 1731468717057Initializing all the Stores at 1731468717058 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717058Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717058Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717058Obtaining lock to block concurrent updates at 1731468717128 (+70 ms)Preparing flush snapshotting stores in 7869ac0070eeb7f4c6fae43c7d0818f2 at 1731468717128Finished memstore snapshotting test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1731468717128Flushing stores of test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. at 1731468717128Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/a: creating writer at 1731468717128Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/a: appending metadata at 1731468717154 (+26 ms)Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/a: closing flushed file at 1731468717154Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/b: creating writer at 1731468717170 (+16 ms)Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/b: appending metadata at 1731468717189 (+19 ms)Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/b: closing flushed file at 1731468717189Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/c: creating writer at 1731468717203 (+14 ms)Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/c: appending metadata at 1731468717228 (+25 ms)Flushing 7869ac0070eeb7f4c6fae43c7d0818f2/c: closing flushed file at 1731468717228Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a375e1a: reopening flushed file at 1731468717241 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@577851a4: reopening flushed file at 1731468717249 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@283c368b: reopening flushed file at 1731468717257 (+8 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for 7869ac0070eeb7f4c6fae43c7d0818f2 in 137ms, sequenceid=6000, compaction requested=false; wal=null at 1731468717265 (+8 ms)Cleaning up temporary data from old regions at 1731468717268 (+3 ms)Region opened successfully at 1731468717274 (+6 ms) 2024-11-13T03:31:57,276 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-11-13T03:31:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 7869ac0070eeb7f4c6fae43c7d0818f2, disabling compactions & flushes 2024-11-13T03:31:57,276 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. after waiting 0 ms 2024-11-13T03:31:57,276 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:57,277 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1731468716365.7869ac0070eeb7f4c6fae43c7d0818f2. 2024-11-13T03:31:57,277 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 7869ac0070eeb7f4c6fae43c7d0818f2: Waiting for close lock at 1731468717276Disabling compacts and flushes for region at 1731468717276Disabling writes for close at 1731468717276Writing region close event to WAL at 1731468717277 (+1 ms)Closed at 1731468717277 2024-11-13T03:31:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741870_1047 (size=93) 2024-11-13T03:31:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741870_1047 (size=93) 2024-11-13T03:31:57,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741870_1047 (size=93) 2024-11-13T03:31:57,286 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:57,286 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468717026) 2024-11-13T03:31:57,298 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#test2727 Thread=404 (was 401) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:47222 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38236 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40290 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:47230 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=929 (was 865) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=2982 (was 3149) 2024-11-13T03:31:57,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=404, OpenFileDescriptor=929, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=2981 2024-11-13T03:31:57,324 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:57,330 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:57,330 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1731468717330 2024-11-13T03:31:57,341 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 2024-11-13T03:31:57,344 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:31:57,348 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 3c3f9579f7c9a34c0a2c1c99ab0632f0, NAME => 'testSequentialEditLogSeqNum,,1731468717324.3c3f9579f7c9a34c0a2c1c99ab0632f0.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:57,349 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1731468717324.3c3f9579f7c9a34c0a2c1c99ab0632f0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:57,349 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,349 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,350 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0 doesn't exist for region: 3c3f9579f7c9a34c0a2c1c99ab0632f0 on table testSequentialEditLogSeqNum 2024-11-13T03:31:57,351 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: 3c3f9579f7c9a34c0a2c1c99ab0632f0 on table testSequentialEditLogSeqNum 2024-11-13T03:31:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:31:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:31:57,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741875_1052 (size=62) 2024-11-13T03:31:57,363 INFO [StoreOpener-3c3f9579f7c9a34c0a2c1c99ab0632f0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,365 INFO [StoreOpener-3c3f9579f7c9a34c0a2c1c99ab0632f0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3c3f9579f7c9a34c0a2c1c99ab0632f0 columnFamilyName a 2024-11-13T03:31:57,365 DEBUG [StoreOpener-3c3f9579f7c9a34c0a2c1c99ab0632f0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,365 INFO [StoreOpener-3c3f9579f7c9a34c0a2c1c99ab0632f0-1 {}] regionserver.HStore(327): Store=3c3f9579f7c9a34c0a2c1c99ab0632f0/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,366 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,367 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,367 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,367 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,369 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 3c3f9579f7c9a34c0a2c1c99ab0632f0 2024-11-13T03:31:57,372 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:57,372 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 3c3f9579f7c9a34c0a2c1c99ab0632f0; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70602709, jitterRate=0.052062347531318665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:31:57,373 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 3c3f9579f7c9a34c0a2c1c99ab0632f0: Writing region info on filesystem at 1731468717349Initializing all the Stores at 1731468717363 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717363Cleaning up temporary data from old regions at 1731468717367 (+4 ms)Region opened successfully at 1731468717373 (+6 ms) 2024-11-13T03:31:57,385 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3c3f9579f7c9a34c0a2c1c99ab0632f0 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-11-13T03:31:57,405 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/.tmp/a/dedbb108cb634be6ae12039b3ace79a7 is 81, key is testSequentialEditLogSeqNum/a:x0/1731468717373/Put/seqid=0 2024-11-13T03:31:57,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:31:57,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:31:57,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741876_1053 (size=5833) 2024-11-13T03:31:57,412 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/.tmp/a/dedbb108cb634be6ae12039b3ace79a7 2024-11-13T03:31:57,420 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/.tmp/a/dedbb108cb634be6ae12039b3ace79a7 as hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/a/dedbb108cb634be6ae12039b3ace79a7 2024-11-13T03:31:57,428 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/a/dedbb108cb634be6ae12039b3ace79a7, entries=10, sequenceid=13, filesize=5.7 K 2024-11-13T03:31:57,429 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 3c3f9579f7c9a34c0a2c1c99ab0632f0 in 44ms, sequenceid=13, compaction requested=false 2024-11-13T03:31:57,430 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3c3f9579f7c9a34c0a2c1c99ab0632f0: 2024-11-13T03:31:57,436 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:31:57,436 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:31:57,437 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:31:57,437 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:31:57,437 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-11-13T03:31:57,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741874_1051 (size=3017) 2024-11-13T03:31:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741874_1051 (size=3017) 2024-11-13T03:31:57,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741874_1051 (size=3017) 2024-11-13T03:31:57,457 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330, size=2.9 K (3017bytes) 2024-11-13T03:31:57,457 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 2024-11-13T03:31:57,458 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 after 1ms 2024-11-13T03:31:57,461 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 took 4ms 2024-11-13T03:31:57,463 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 so closing down 2024-11-13T03:31:57,463 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:31:57,464 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468717330.temp 2024-11-13T03:31:57,466 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp 2024-11-13T03:31:57,466 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:31:57,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741877_1054 (size=2433) 2024-11-13T03:31:57,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741877_1054 (size=2433) 2024-11-13T03:31:57,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741877_1054 (size=2433) 2024-11-13T03:31:57,476 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T03:31:57,478 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp to hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000020 2024-11-13T03:31:57,479 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330, size=2.9 K, length=3017, corrupted=false, cancelled=false 2024-11-13T03:31:57,479 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330, journal: Splitting hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330, size=2.9 K (3017bytes) at 1731468717457Finishing writing output for hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330 so closing down at 1731468717463 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp at 1731468717466 (+3 ms)3 split writer threads finished at 1731468717466Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731468717477 (+11 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000003-wal.1731468717330.temp to hdfs://localhost:33705/hbase/data/default/testSequentialEditLogSeqNum/3c3f9579f7c9a34c0a2c1c99ab0632f0/recovered.edits/0000000000000000020 at 1731468717478 (+1 ms)Processed 17 edits across 1 Regions in 18 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1731468717323/wal.1731468717330, size=2.9 K, length=3017, corrupted=false, cancelled=false at 1731468717479 (+1 ms) 2024-11-13T03:31:57,494 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testSequentialEditLogSeqNum Thread=409 (was 404) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:47222 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38236 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:47230 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=967 (was 929) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=335 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=2971 (was 2981) 2024-11-13T03:31:57,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=409, OpenFileDescriptor=967, MaxFileDescriptor=1048576, SystemLoadAverage=335, ProcessCount=11, AvailableMemoryMB=2970 2024-11-13T03:31:57,523 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:57,526 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:31:57,549 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:31:57,552 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-02077930, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-02077930, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:57,565 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-02077930/hregion-02077930.1731468717553, exclude list is [], retry=0 2024-11-13T03:31:57,568 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:57,568 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:57,569 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:57,571 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-02077930/hregion-02077930.1731468717553 2024-11-13T03:31:57,572 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:31:57,572 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 94c6a02e27d5454871a93e205115a6a3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:31:57,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:31:57,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:31:57,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741879_1056 (size=70) 2024-11-13T03:31:57,587 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:57,589 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,590 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName a 2024-11-13T03:31:57,591 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,591 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,591 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,593 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName b 2024-11-13T03:31:57,593 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,593 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,594 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,595 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName c 2024-11-13T03:31:57,595 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,596 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,596 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,597 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,598 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,599 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:31:57,600 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,602 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:31:57,603 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94c6a02e27d5454871a93e205115a6a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66730015, jitterRate=-0.005645290017127991}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94c6a02e27d5454871a93e205115a6a3: Writing region info on filesystem at 1731468717587Initializing all the Stores at 1731468717588 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717588Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717588Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717588Cleaning up temporary data from old regions at 1731468717598 (+10 ms)Region opened successfully at 1731468717603 (+5 ms) 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94c6a02e27d5454871a93e205115a6a3, disabling compactions & flushes 2024-11-13T03:31:57,604 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. after waiting 0 ms 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:31:57,604 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:31:57,604 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94c6a02e27d5454871a93e205115a6a3: Waiting for close lock at 1731468717604Disabling compacts and flushes for region at 1731468717604Disabling writes for close at 1731468717604Writing region close event to WAL at 1731468717604Closed at 1731468717604 2024-11-13T03:31:57,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741878_1055 (size=93) 2024-11-13T03:31:57,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741878_1055 (size=93) 2024-11-13T03:31:57,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741878_1055 (size=93) 2024-11-13T03:31:57,608 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-02077930/hregion-02077930.1731468717553 not finished, retry = 0 2024-11-13T03:31:57,711 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:31:57,711 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-02077930:(num 1731468717553) 2024-11-13T03:31:57,711 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:31:57,713 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:31:57,726 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, exclude list is [], retry=0 2024-11-13T03:31:57,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:31:57,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:31:57,729 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:31:57,735 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 2024-11-13T03:31:57,736 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:31:57,736 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c6a02e27d5454871a93e205115a6a3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:31:57,736 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:31:57,736 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,737 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,740 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,741 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName a 2024-11-13T03:31:57,741 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,741 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,741 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,742 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName b 2024-11-13T03:31:57,742 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,742 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,743 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,743 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName c 2024-11-13T03:31:57,743 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:31:57,744 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:31:57,744 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,744 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,745 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,747 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,747 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,748 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:31:57,749 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:31:57,750 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94c6a02e27d5454871a93e205115a6a3; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66963528, jitterRate=-0.002165675163269043}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:31:57,752 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94c6a02e27d5454871a93e205115a6a3: Writing region info on filesystem at 1731468717737Initializing all the Stores at 1731468717738 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717738Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717739 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468717739Cleaning up temporary data from old regions at 1731468717747 (+8 ms)Region opened successfully at 1731468717752 (+5 ms) 2024-11-13T03:31:57,757 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1731468717755/Put/seqid=0 2024-11-13T03:31:57,766 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:31:57,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:31:57,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741881_1058 (size=4826) 2024-11-13T03:31:57,768 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33705/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 94c6a02e27d5454871a93e205115a6a3/a 2024-11-13T03:31:57,777 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-11-13T03:31:57,777 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:31:57,778 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 94c6a02e27d5454871a93e205115a6a3: 2024-11-13T03:31:57,780 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/599b57055e674eda9f62bead662aa14d_SeqId_3_ 2024-11-13T03:31:57,781 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33705/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 94c6a02e27d5454871a93e205115a6a3/a as hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/599b57055e674eda9f62bead662aa14d_SeqId_3_ - updating store file list. 2024-11-13T03:31:57,788 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 599b57055e674eda9f62bead662aa14d_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T03:31:57,788 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/599b57055e674eda9f62bead662aa14d_SeqId_3_ into 94c6a02e27d5454871a93e205115a6a3/a 2024-11-13T03:31:57,788 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33705/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 94c6a02e27d5454871a93e205115a6a3/a (new location: hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/599b57055e674eda9f62bead662aa14d_SeqId_3_) 2024-11-13T03:31:57,844 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, size=0 (0bytes) 2024-11-13T03:31:57,844 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 might be still open, length is 0 2024-11-13T03:31:57,844 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 2024-11-13T03:31:57,844 WARN [IPC Server handler 4 on default port 33705 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-11-13T03:31:57,845 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 after 1ms 2024-11-13T03:31:57,943 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:47410 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47410 dst: /127.0.0.1:43451 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43451 remote=/127.0.0.1:47410]. Total timeout mills is 60000, 59851 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:57,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38342 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:43285:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38342 dst: /127.0.0.1:43285 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:57,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40396 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:37801:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:40396 dst: /127.0.0.1:37801 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:31:57,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741880_1059 (size=504) 2024-11-13T03:31:57,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741880_1059 (size=504) 2024-11-13T03:31:57,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741880_1059 (size=504) 2024-11-13T03:32:01,846 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 after 4002ms 2024-11-13T03:32:01,855 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 took 4010ms 2024-11-13T03:32:01,857 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714; continuing. 2024-11-13T03:32:01,857 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 so closing down 2024-11-13T03:32:01,857 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:01,859 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1731468717714.temp 2024-11-13T03:32:01,860 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp 2024-11-13T03:32:01,861 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741882_1060 (size=238) 2024-11-13T03:32:01,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741882_1060 (size=238) 2024-11-13T03:32:01,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741882_1060 (size=238) 2024-11-13T03:32:01,867 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:01,868 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp to hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 2024-11-13T03:32:01,868 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:32:01,868 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, journal: Splitting hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, size=0 (0bytes) at 1731468717844Finishing writing output for hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 so closing down at 1731468721857 (+4013 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp at 1731468721861 (+4 ms)3 split writer threads finished at 1731468721861Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1731468721867 (+6 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005-wal.1731468717714.temp to hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 at 1731468721868 (+1 ms)Processed 2 edits across 1 Regions in 13 ms; skipped=1; WAL=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714, size=0, length=0, corrupted=false, cancelled=false at 1731468721868 2024-11-13T03:32:01,870 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468717714 2024-11-13T03:32:01,870 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 2024-11-13T03:32:01,870 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:01,872 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:01,885 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468721872, exclude list is [], retry=0 2024-11-13T03:32:01,887 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:01,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:01,888 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:01,889 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468721872 2024-11-13T03:32:01,890 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:01,890 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 94c6a02e27d5454871a93e205115a6a3, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:01,890 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:01,890 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,890 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,892 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,893 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName a 2024-11-13T03:32:01,893 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:01,899 DEBUG [StoreFileOpener-94c6a02e27d5454871a93e205115a6a3-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 599b57055e674eda9f62bead662aa14d_SeqId_3_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:01,899 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/599b57055e674eda9f62bead662aa14d_SeqId_3_ 2024-11-13T03:32:01,900 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:01,900 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,901 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName b 2024-11-13T03:32:01,901 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:01,901 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:01,901 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,902 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 94c6a02e27d5454871a93e205115a6a3 columnFamilyName c 2024-11-13T03:32:01,902 DEBUG [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:01,903 INFO [StoreOpener-94c6a02e27d5454871a93e205115a6a3-1 {}] regionserver.HStore(327): Store=94c6a02e27d5454871a93e205115a6a3/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:01,903 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,904 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,905 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,906 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 2024-11-13T03:32:01,909 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 2024-11-13T03:32:01,909 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 94c6a02e27d5454871a93e205115a6a3 3/3 column families, dataSize=58 B heapSize=904 B 2024-11-13T03:32:01,922 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/.tmp/a/a0baee71757342378b4e4121e501c63b is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1731468717794/Put/seqid=0 2024-11-13T03:32:01,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:01,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:01,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741884_1062 (size=5149) 2024-11-13T03:32:01,929 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/.tmp/a/a0baee71757342378b4e4121e501c63b 2024-11-13T03:32:01,934 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/.tmp/a/a0baee71757342378b4e4121e501c63b as hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/a0baee71757342378b4e4121e501c63b 2024-11-13T03:32:01,939 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/a/a0baee71757342378b4e4121e501c63b, entries=1, sequenceid=5, filesize=5.0 K 2024-11-13T03:32:01,939 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 94c6a02e27d5454871a93e205115a6a3 in 30ms, sequenceid=5, compaction requested=false; wal=null 2024-11-13T03:32:01,939 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/0000000000000000005 2024-11-13T03:32:01,941 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,941 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,941 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:01,943 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 94c6a02e27d5454871a93e205115a6a3 2024-11-13T03:32:01,945 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/94c6a02e27d5454871a93e205115a6a3/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-11-13T03:32:01,946 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 94c6a02e27d5454871a93e205115a6a3; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74654054, jitterRate=0.1124320924282074}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:01,946 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 94c6a02e27d5454871a93e205115a6a3: Writing region info on filesystem at 1731468721891Initializing all the Stores at 1731468721891Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468721891Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468721892 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468721892Obtaining lock to block concurrent updates at 1731468721909 (+17 ms)Preparing flush snapshotting stores in 94c6a02e27d5454871a93e205115a6a3 at 1731468721909Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1731468721909Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. at 1731468721909Flushing 94c6a02e27d5454871a93e205115a6a3/a: creating writer at 1731468721909Flushing 94c6a02e27d5454871a93e205115a6a3/a: appending metadata at 1731468721922 (+13 ms)Flushing 94c6a02e27d5454871a93e205115a6a3/a: closing flushed file at 1731468721922Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@17a96ec9: reopening flushed file at 1731468721933 (+11 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 94c6a02e27d5454871a93e205115a6a3 in 30ms, sequenceid=5, compaction requested=false; wal=null at 1731468721939 (+6 ms)Cleaning up temporary data from old regions at 1731468721941 (+2 ms)Region opened successfully at 1731468721946 (+5 ms) 2024-11-13T03:32:01,950 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 94c6a02e27d5454871a93e205115a6a3, disabling compactions & flushes 2024-11-13T03:32:01,950 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:32:01,950 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:32:01,950 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. after waiting 0 ms 2024-11-13T03:32:01,950 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:32:01,951 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1731468717524.94c6a02e27d5454871a93e205115a6a3. 2024-11-13T03:32:01,951 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 94c6a02e27d5454871a93e205115a6a3: Waiting for close lock at 1731468721950Disabling compacts and flushes for region at 1731468721950Disabling writes for close at 1731468721950Writing region close event to WAL at 1731468721951 (+1 ms)Closed at 1731468721951 2024-11-13T03:32:01,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741883_1061 (size=93) 2024-11-13T03:32:01,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741883_1061 (size=93) 2024-11-13T03:32:01,954 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741883_1061 (size=93) 2024-11-13T03:32:01,956 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:01,956 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468721872) 2024-11-13T03:32:01,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894685881_22 at /127.0.0.1:54020 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:33705 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894685881_22 at /127.0.0.1:58996 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33705 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1894685881_22 at /127.0.0.1:40416 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1025 (was 967) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=316 (was 335), ProcessCount=11 (was 11), AvailableMemoryMB=2965 (was 2970) 2024-11-13T03:32:01,971 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T03:32:01,981 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1025, MaxFileDescriptor=1048576, SystemLoadAverage=316, ProcessCount=11, AvailableMemoryMB=2965 2024-11-13T03:32:01,981 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1025 is superior to 1024 2024-11-13T03:32:01,996 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:02,000 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-13T03:32:02,004 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 8394b182524b,41967,1731468706868 2024-11-13T03:32:02,006 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a8673d9 2024-11-13T03:32:02,007 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-13T03:32:02,009 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36046, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-13T03:32:02,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-13T03:32:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T03:32:02,020 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-11-13T03:32:02,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-11-13T03:32:02,023 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:02,025 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-13T03:32:02,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:02,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:02,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741885_1063 (size=694) 2024-11-13T03:32:02,038 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda 2024-11-13T03:32:02,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:02,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:02,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741886_1064 (size=77) 2024-11-13T03:32:02,046 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:02,047 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:02,047 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,047 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,047 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:02,047 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,047 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,047 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468722047Disabling compacts and flushes for region at 1731468722047Disabling writes for close at 1731468722047Writing region close event to WAL at 1731468722047Closed at 1731468722047 2024-11-13T03:32:02,048 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-11-13T03:32:02,053 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1731468722049"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731468722049"}]},"ts":"1731468722049"} 2024-11-13T03:32:02,056 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-13T03:32:02,057 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-13T03:32:02,060 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731468722058"}]},"ts":"1731468722058"} 2024-11-13T03:32:02,064 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-11-13T03:32:02,065 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-11-13T03:32:02,066 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:02,066 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:02,066 INFO [PEWorker-3 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-11-13T03:32:02,066 DEBUG [PEWorker-3 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:02,068 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN}] 2024-11-13T03:32:02,070 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN 2024-11-13T03:32:02,071 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN; state=OFFLINE, location=8394b182524b,40261,1731468707776; forceNewPlan=false, retain=false 2024-11-13T03:32:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:02,224 INFO [8394b182524b:41967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:02,225 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPENING, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:32:02,228 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN because future has completed 2024-11-13T03:32:02,229 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776}] 2024-11-13T03:32:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:02,390 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,390 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:02,391 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,391 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:02,392 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,392 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,393 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,395 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:02,395 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:02,396 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:02,396 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,398 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:02,398 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:02,399 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:02,399 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,400 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,400 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,401 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,401 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,402 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:02,403 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,405 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:02,406 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened fa8df6bead70f7386f753fb4aa9c8270; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69972036, jitterRate=0.04266458749771118}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:02,406 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,407 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for fa8df6bead70f7386f753fb4aa9c8270: Running coprocessor pre-open hook at 1731468722392Writing region info on filesystem at 1731468722392Initializing all the Stores at 1731468722393 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468722393Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468722393Cleaning up temporary data from old regions at 1731468722401 (+8 ms)Running coprocessor post-open hooks at 1731468722406 (+5 ms)Region opened successfully at 1731468722407 (+1 ms) 2024-11-13T03:32:02,408 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., pid=6, masterSystemTime=1731468722381 2024-11-13T03:32:02,411 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,411 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,411 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPEN, openSeqNum=2, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:32:02,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 because future has completed 2024-11-13T03:32:02,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-13T03:32:02,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 in 187 msec 2024-11-13T03:32:02,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-13T03:32:02,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN in 350 msec 2024-11-13T03:32:02,424 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-13T03:32:02,424 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731468722424"}]},"ts":"1731468722424"} 2024-11-13T03:32:02,427 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-11-13T03:32:02,428 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-11-13T03:32:02,430 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 414 msec 2024-11-13T03:32:02,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-13T03:32:02,660 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-11-13T03:32:02,660 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-11-13T03:32:02,661 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T03:32:02,668 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-11-13T03:32:02,669 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-11-13T03:32:02,670 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-11-13T03:32:02,682 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=2] 2024-11-13T03:32:02,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=fa8df6bead70f7386f753fb4aa9c8270, source=8394b182524b,40261,1731468707776, destination=8394b182524b,43461,1731468707627, warming up region on 8394b182524b,43461,1731468707627 2024-11-13T03:32:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:02,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=fa8df6bead70f7386f753fb4aa9c8270, source=8394b182524b,40261,1731468707776, destination=8394b182524b,43461,1731468707627, running balancer 2024-11-13T03:32:02,702 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56345, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE 2024-11-13T03:32:02,703 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE 2024-11-13T03:32:02,705 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=CLOSING, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:32:02,708 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE because future has completed 2024-11-13T03:32:02,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(7855): Warmup {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:02,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:02,711 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:32:02,711 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776}] 2024-11-13T03:32:02,712 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:02,712 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:02,713 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:02,713 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,715 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:02,715 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:02,715 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:02,716 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:02,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,717 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468722716Disabling compacts and flushes for region at 1731468722716Disabling writes for close at 1731468722716Writing region close event to WAL at 1731468722717 (+1 ms)Closed at 1731468722717 2024-11-13T03:32:02,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-11-13T03:32:02,870 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,871 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:32:02,872 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:02,872 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,872 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,872 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:02,872 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,872 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing fa8df6bead70f7386f753fb4aa9c8270 2/2 column families, dataSize=31 B heapSize=616 B 2024-11-13T03:32:02,894 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/9528de6bef5b45c3adcf211d7af03e79 is 35, key is r1/cf1:q/1731468722685/Put/seqid=0 2024-11-13T03:32:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:02,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741887_1065 (size=4783) 2024-11-13T03:32:02,903 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/9528de6bef5b45c3adcf211d7af03e79 2024-11-13T03:32:02,910 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/9528de6bef5b45c3adcf211d7af03e79 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79 2024-11-13T03:32:02,917 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79, entries=1, sequenceid=5, filesize=4.7 K 2024-11-13T03:32:02,918 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for fa8df6bead70f7386f753fb4aa9c8270 in 46ms, sequenceid=5, compaction requested=false 2024-11-13T03:32:02,918 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-11-13T03:32:02,924 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T03:32:02,926 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:02,926 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468722871Running coprocessor pre-close hooks at 1731468722871Disabling compacts and flushes for region at 1731468722871Disabling writes for close at 1731468722872 (+1 ms)Obtaining lock to block concurrent updates at 1731468722872Preparing flush snapshotting stores in fa8df6bead70f7386f753fb4aa9c8270 at 1731468722872Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1731468722873 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. at 1731468722874 (+1 ms)Flushing fa8df6bead70f7386f753fb4aa9c8270/cf1: creating writer at 1731468722875 (+1 ms)Flushing fa8df6bead70f7386f753fb4aa9c8270/cf1: appending metadata at 1731468722894 (+19 ms)Flushing fa8df6bead70f7386f753fb4aa9c8270/cf1: closing flushed file at 1731468722894Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71cf6bb0: reopening flushed file at 1731468722909 (+15 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for fa8df6bead70f7386f753fb4aa9c8270 in 46ms, sequenceid=5, compaction requested=false at 1731468722918 (+9 ms)Writing region close event to WAL at 1731468722920 (+2 ms)Running coprocessor post-close hooks at 1731468722925 (+5 ms)Closed at 1731468722926 (+1 ms) 2024-11-13T03:32:02,927 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding fa8df6bead70f7386f753fb4aa9c8270 move to 8394b182524b,43461,1731468707627 record at close sequenceid=5 2024-11-13T03:32:02,930 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:02,931 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=CLOSED 2024-11-13T03:32:02,934 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 because future has completed 2024-11-13T03:32:02,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-13T03:32:02,940 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 in 225 msec 2024-11-13T03:32:02,942 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE; state=CLOSED, location=8394b182524b,43461,1731468707627; forceNewPlan=false, retain=false 2024-11-13T03:32:03,093 INFO [8394b182524b:41967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:03,093 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPENING, regionLocation=8394b182524b,43461,1731468707627 2024-11-13T03:32:03,099 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE because future has completed 2024-11-13T03:32:03,100 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627}] 2024-11-13T03:32:03,264 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,264 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:03,265 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,265 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:03,265 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,266 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,269 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,270 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:03,270 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:03,278 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79 2024-11-13T03:32:03,279 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:03,279 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,280 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:03,280 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:03,281 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:03,281 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,282 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,284 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,285 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,285 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,286 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:03,288 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,290 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened fa8df6bead70f7386f753fb4aa9c8270; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63858210, jitterRate=-0.04843851923942566}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:03,290 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,291 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for fa8df6bead70f7386f753fb4aa9c8270: Running coprocessor pre-open hook at 1731468723266Writing region info on filesystem at 1731468723266Initializing all the Stores at 1731468723268 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468723268Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468723268Cleaning up temporary data from old regions at 1731468723285 (+17 ms)Running coprocessor post-open hooks at 1731468723290 (+5 ms)Region opened successfully at 1731468723291 (+1 ms) 2024-11-13T03:32:03,293 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., pid=9, masterSystemTime=1731468723255 2024-11-13T03:32:03,296 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,296 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,296 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPEN, openSeqNum=9, regionLocation=8394b182524b,43461,1731468707627 2024-11-13T03:32:03,300 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627 because future has completed 2024-11-13T03:32:03,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-11-13T03:32:03,305 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627 in 202 msec 2024-11-13T03:32:03,307 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE in 604 msec 2024-11-13T03:32:03,359 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:03,361 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44044, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:03,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:33222 deadline: 1731468783364, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=43461 startCode=1731468707627. As of locationSeqNum=5. 2024-11-13T03:32:03,386 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=43461 startCode=1731468707627. As of locationSeqNum=5. 2024-11-13T03:32:03,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=43461 startCode=1731468707627. As of locationSeqNum=5. 2024-11-13T03:32:03,387 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,43461,1731468707627, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=43461 startCode=1731468707627. As of locationSeqNum=5. 2024-11-13T03:32:03,498 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:03,500 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44054, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:03,510 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing fa8df6bead70f7386f753fb4aa9c8270 2/2 column families, dataSize=50 B heapSize=720 B 2024-11-13T03:32:03,529 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/20033efe252742a4af109b8dd3a6d575 is 29, key is r1/cf1:/1731468723502/DeleteFamily/seqid=0 2024-11-13T03:32:03,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:03,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:03,537 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741888_1066 (size=4906) 2024-11-13T03:32:03,542 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,557 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf2/79a4e9f929f243049d1a983835da3cfa is 29, key is r1/cf2:/1731468723502/DeleteFamily/seqid=0 2024-11-13T03:32:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:03,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:03,564 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741889_1067 (size=4906) 2024-11-13T03:32:03,564 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf2/79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,570 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,571 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/20033efe252742a4af109b8dd3a6d575 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,578 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,578 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T03:32:03,579 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf2/79a4e9f929f243049d1a983835da3cfa as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,586 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,586 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa, entries=1, sequenceid=12, filesize=4.8 K 2024-11-13T03:32:03,587 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for fa8df6bead70f7386f753fb4aa9c8270 in 77ms, sequenceid=12, compaction requested=false 2024-11-13T03:32:03,587 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for fa8df6bead70f7386f753fb4aa9c8270: 2024-11-13T03:32:03,589 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-13T03:32:03,591 DEBUG [Time-limited test {}] regionserver.HStore(1541): fa8df6bead70f7386f753fb4aa9c8270/cf1 is initiating major compaction (all files) 2024-11-13T03:32:03,591 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:03,591 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:03,591 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fa8df6bead70f7386f753fb4aa9c8270/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,592 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79, hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575] into tmpdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp, totalSize=9.5 K 2024-11-13T03:32:03,593 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 9528de6bef5b45c3adcf211d7af03e79, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1731468722685 2024-11-13T03:32:03,593 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 20033efe252742a4af109b8dd3a6d575, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T03:32:03,605 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): fa8df6bead70f7386f753fb4aa9c8270#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:32:03,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:03,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:03,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741890_1068 (size=4626) 2024-11-13T03:32:03,620 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf1/c59df0d954784ea2b48247cef05c723c as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/c59df0d954784ea2b48247cef05c723c 2024-11-13T03:32:03,633 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in fa8df6bead70f7386f753fb4aa9c8270/cf1 of fa8df6bead70f7386f753fb4aa9c8270 into c59df0d954784ea2b48247cef05c723c(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:32:03,633 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fa8df6bead70f7386f753fb4aa9c8270: 2024-11-13T03:32:03,633 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-13T03:32:03,633 DEBUG [Time-limited test {}] regionserver.HStore(1541): fa8df6bead70f7386f753fb4aa9c8270/cf2 is initiating major compaction (all files) 2024-11-13T03:32:03,633 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-13T03:32:03,633 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-13T03:32:03,633 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of fa8df6bead70f7386f753fb4aa9c8270/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,634 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa] into tmpdir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp, totalSize=4.8 K 2024-11-13T03:32:03,634 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 79a4e9f929f243049d1a983835da3cfa, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-11-13T03:32:03,640 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): fa8df6bead70f7386f753fb4aa9c8270#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-13T03:32:03,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:03,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741891_1069 (size=4592) 2024-11-13T03:32:03,653 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/.tmp/cf2/dc04e3dce7c448c585484514a61bc486 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/dc04e3dce7c448c585484514a61bc486 2024-11-13T03:32:03,660 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in fa8df6bead70f7386f753fb4aa9c8270/cf2 of fa8df6bead70f7386f753fb4aa9c8270 into dc04e3dce7c448c585484514a61bc486(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:32:03,660 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for fa8df6bead70f7386f753fb4aa9c8270: 2024-11-13T03:32:03,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=fa8df6bead70f7386f753fb4aa9c8270, source=8394b182524b,43461,1731468707627, destination=8394b182524b,40261,1731468707776, warming up region on 8394b182524b,40261,1731468707776 2024-11-13T03:32:03,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=fa8df6bead70f7386f753fb4aa9c8270, source=8394b182524b,43461,1731468707627, destination=8394b182524b,40261,1731468707776, running balancer 2024-11-13T03:32:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE 2024-11-13T03:32:03,668 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE 2024-11-13T03:32:03,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,669 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=CLOSING, regionLocation=8394b182524b,43461,1731468707627 2024-11-13T03:32:03,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(7855): Warmup {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:03,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:03,670 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,671 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:03,671 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:03,672 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE because future has completed 2024-11-13T03:32:03,673 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-13T03:32:03,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627}] 2024-11-13T03:32:03,679 INFO [StoreFileOpener-fa8df6bead70f7386f753fb4aa9c8270-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,679 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,684 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79 2024-11-13T03:32:03,690 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/c59df0d954784ea2b48247cef05c723c 2024-11-13T03:32:03,690 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:03,690 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,692 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:03,692 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:03,698 INFO [StoreFileOpener-fa8df6bead70f7386f753fb4aa9c8270-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,699 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,705 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/dc04e3dce7c448c585484514a61bc486 2024-11-13T03:32:03,705 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:03,705 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:03,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40261 {}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468723705Disabling compacts and flushes for region at 1731468723705Disabling writes for close at 1731468723705Writing region close event to WAL at 1731468723708 (+3 ms)Closed at 1731468723708 2024-11-13T03:32:03,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-11-13T03:32:03,827 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,827 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-13T03:32:03,827 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:03,828 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,828 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,828 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:03,828 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,828 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79, hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575] to archive 2024-11-13T03:32:03,831 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:32:03,834 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79 to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/9528de6bef5b45c3adcf211d7af03e79 2024-11-13T03:32:03,836 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575 to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/20033efe252742a4af109b8dd3a6d575 2024-11-13T03:32:03,848 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa] to archive 2024-11-13T03:32:03,849 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:32:03,851 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/79a4e9f929f243049d1a983835da3cfa 2024-11-13T03:32:03,856 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-11-13T03:32:03,857 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:03,857 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468723827Running coprocessor pre-close hooks at 1731468723827Disabling compacts and flushes for region at 1731468723827Disabling writes for close at 1731468723828 (+1 ms)Writing region close event to WAL at 1731468723852 (+24 ms)Running coprocessor post-close hooks at 1731468723857 (+5 ms)Closed at 1731468723857 2024-11-13T03:32:03,857 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding fa8df6bead70f7386f753fb4aa9c8270 move to 8394b182524b,40261,1731468707776 record at close sequenceid=12 2024-11-13T03:32:03,860 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:03,861 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=CLOSED 2024-11-13T03:32:03,863 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627 because future has completed 2024-11-13T03:32:03,867 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-11-13T03:32:03,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,43461,1731468707627 in 192 msec 2024-11-13T03:32:03,868 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE; state=CLOSED, location=8394b182524b,40261,1731468707776; forceNewPlan=false, retain=false 2024-11-13T03:32:04,019 INFO [8394b182524b:41967 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-11-13T03:32:04,020 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPENING, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:32:04,029 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE because future has completed 2024-11-13T03:32:04,030 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776}] 2024-11-13T03:32:04,187 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,187 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:04,188 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,188 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:04,188 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,188 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,190 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,192 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:04,192 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:04,200 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/c59df0d954784ea2b48247cef05c723c 2024-11-13T03:32:04,200 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:04,201 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,202 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:04,202 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:04,211 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/dc04e3dce7c448c585484514a61bc486 2024-11-13T03:32:04,212 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:04,212 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,213 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,214 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,215 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,215 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,216 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:04,217 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,218 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened fa8df6bead70f7386f753fb4aa9c8270; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69610344, jitterRate=0.037274956703186035}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:04,218 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,219 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for fa8df6bead70f7386f753fb4aa9c8270: Running coprocessor pre-open hook at 1731468724189Writing region info on filesystem at 1731468724189Initializing all the Stores at 1731468724190 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468724190Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468724190Cleaning up temporary data from old regions at 1731468724215 (+25 ms)Running coprocessor post-open hooks at 1731468724218 (+3 ms)Region opened successfully at 1731468724219 (+1 ms) 2024-11-13T03:32:04,220 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., pid=12, masterSystemTime=1731468724183 2024-11-13T03:32:04,222 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,222 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,223 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPEN, openSeqNum=18, regionLocation=8394b182524b,40261,1731468707776 2024-11-13T03:32:04,226 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 because future has completed 2024-11-13T03:32:04,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-11-13T03:32:04,230 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,40261,1731468707776 in 198 msec 2024-11-13T03:32:04,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, REOPEN/MOVE in 564 msec 2024-11-13T03:32:04,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:04,274 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:41260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:04,277 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server 8394b182524b,40261,1731468707776: testing ***** 2024-11-13T03:32:04,278 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-11-13T03:32:04,282 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-11-13T03:32:04,284 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-11-13T03:32:04,287 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-11-13T03:32:04,288 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-11-13T03:32:04,297 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 416400896 }, "NonHeapMemoryUsage": { "committed": 170459136, "init": 7667712, "max": -1, "used": 167883504 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 0, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 2071, "exceptions.RegionMovedException": 0, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 8, "ProcessCallTime_min": 0, "ProcessCallTime_max": 12, "ProcessCallTime_mean": 4, "ProcessCallTime_25th_percentile": 3, "ProcessCallTime_median": 6, "ProcessCallTime_75th_percentile": 9, "ProcessCallTime_90th_percentile": 10, "ProcessCallTime_95th_percentile": 11, "ProcessCallTime_98th_percentile": 11, "ProcessCallTime_99th_percentile": 11, "ProcessCallTime_99.9th_percentile": 11, "ProcessCallTime_TimeRangeCount_0-1": 8, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 8, "QueueCallTime_min": 0, "QueueCallTime_max": 1, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 0, "QueueCallTime_95th_percentile": 0, "QueueCallTime_98th_percentile": 0, "QueueCallTime_99th_percentile": 0, "QueueCallTime_99.9th_percentile": 0, "QueueCallTime_TimeRangeCount_0-1": 8, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 8, "TotalCallTime_min": 0, "TotalCallTime_max": 12, "TotalCallTime_mean": 4, "TotalCallTime_25th_percentile": 3, "TotalCallTime_median": 6, "TotalCallTime_75th_percentile": 9, "TotalCallTime_90th_percentile": 10, "TotalCallTime_95th_percentile": 11, "TotalCallTime_98th_percentile": 11, "TotalCallTime_99th_percentile": 11, "TotalCallTime_99.9th_percentile": 11, "TotalCallTime_TimeRangeCount_0-1": 8, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 8, "ResponseSize_min": 0, "ResponseSize_max": 174, "ResponseSize_mean": 50, "ResponseSize_25th_percentile": 43, "ResponseSize_median": 87, "ResponseSize_75th_percentile": 130, "ResponseSize_90th_percentile": 156, "ResponseSize_95th_percentile": 165, "ResponseSize_98th_percentile": 170, "ResponseSize_99th_percentile": 172, "ResponseSize_99.9th_percentile": 173, "ResponseSize_SizeRangeCount_0-10": 8, "exceptions.UnknownScannerException": 0, "exceptions": 0, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 8, "RequestSize_min": 89, "RequestSize_max": 121, "RequestSize_mean": 103, "RequestSize_25th_percentile": 97, "RequestSize_median": 105, "RequestSize_75th_percentile": 113, "RequestSize_90th_percentile": 117, "RequestSize_95th_percentile": 119, "RequestSize_98th_percentile": 120, "RequestSize_99th_percentile": 120, "RequestSize_99.9th_percentile": 120, "RequestSize_SizeRangeCount_0-10": 8, "sentBytes": 348 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:58534", "tag.serverName": "8394b182524b,41763,1731468707723", "tag.clusterId": "1ce67cc5-da14-493a-907b-407a287aeb9e", "tag.Context": "regionserver", "tag.Hostname": "8394b182524b", "regionCount": 0, "storeCount": 0, "hlogFileCount": 1, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 0, "memStoreHeapSize": 0, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1731468707723, "averageRegionSize": 0, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 10034, "localBytesRead": 10034, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 0.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 0, "l1CacheMissCount": 0, "l1CacheHitRatio": 0.0, "l1CacheMissRatio": 0.0, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 4782, "ByteBuffAllocatorPoolAllocationBytes": 0, "ByteBuffAllocatorHeapAllocationRatio": 0.0, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 0, "activeScanners": 0, "totalRequestCount": 0, "totalRowActionRequestCount": 0, "readRequestCount": 0, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 0, "rpcGetRequestCount": 0, "rpcFullScanRequestCount": 0, "rpcScanRequestCount": 0, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 0, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 0, "blockCacheHitCountPrimary": 0, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 0, "blockCacheMissCountPrimary": 0, "blockCacheMissCachingCount": 0, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 0, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 0, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 0, "MajorCompactionTime_min": 0, "MajorCompactionTime_max": 0, "MajorCompactionTime_mean": 0, "MajorCompactionTime_25th_percentile": 0, "MajorCompactionTime_median": 0, "MajorCompactionTime_75th_percentile": 0, "MajorCompactionTime_90th_percentile": 0, "MajorCompactionTime_95th_percentile": 0, "MajorCompactionTime_98th_percentile": 0, "MajorCompactionTime_99th_percentile": 0, "MajorCompactionTime_99.9th_percentile": 0, "ScanTime_num_ops": 0, "ScanTime_min": 0, "ScanTime_max": 0, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 0, "ScanTime_75th_percentile": 0, "ScanTime_90th_percentile": 0, "ScanTime_95th_percentile": 0, "ScanTime_98th_percentile": 0, "ScanTime_99th_percentile": 0, "ScanTime_99.9th_percentile": 0, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 0, "Put_min": 0, "Put_max": 0, "Put_mean": 0, "Put_25th_percentile": 0, "Put_median": 0, "Put_75th_percentile": 0, "Put_90th_percentile": 0, "Put_95th_percentile": 0, "Put_98th_percentile": 0, "Put_99th_percentile": 0, "Put_99.9th_percentile": 0, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 0, "slowAppendCount": 0, "flushedOutputBytes": 0, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 0, "MajorCompactionInputSize_min": 0, "MajorCompactionInputSize_max": 0, "MajorCompactionInputSize_mean": 0, "MajorCompactionInputSize_25th_percentile": 0, "MajorCompactionInputSize_median": 0, "MajorCompactionInputSize_75th_percentile": 0, "MajorCompactionInputSize_90th_percentile": 0, "MajorCompactionInputSize_95th_percentile": 0, "MajorCompactionInputSize_98th_percentile": 0, "MajorCompactionInputSize_99th_percentile": 0, "MajorCompactionInputSize_99.9th_percentile": 0, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 0, "CompactionInputSize_min": 0, "CompactionInputSize_max": 0, "CompactionInputSize_mean": 0, "CompactionInputSize_25th_percentile": 0, "CompactionInputSize_median": 0, "CompactionInputSize_75th_percentile": 0, "CompactionInputSize_90th_percentile": 0, "CompactionInputSize_95th_percentile": 0, "CompactionInputSize_98th_percentile": 0, "CompactionInputSize_99th_percentile": 0, "CompactionInputSize_99.9th_percentile": 0, "flushedMemstoreBytes": 0, "majorCompactedOutputBytes": 0, "slowPutCount": 0, "compactedInputBytes": 0, "FlushOutputSize_num_ops": 0, "FlushOutputSize_min": 0, "FlushOutputSize_max": 0, "FlushOutputSize_mean": 0, "FlushOutputSize_25th_percentile": 0, "FlushOutputSize_median": 0, "FlushOutputSize_75th_percentile": 0, "FlushOutputSize_90th_percentile": 0, "FlushOutputSize_95th_percentile": 0, "FlushOutputSize_98th_percentile": 0, "FlushOutputSize_99th_percentile": 0, "FlushOutputSize_99.9th_percentile": 0, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 0, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 0, "ScanBlockBytesScanned_min": 0, "ScanBlockBytesScanned_max": 0, "ScanBlockBytesScanned_mean": 0, "ScanBlockBytesScanned_25th_percentile": 0, "ScanBlockBytesScanned_median": 0, "ScanBlockBytesScanned_75th_percentile": 0, "ScanBlockBytesScanned_90th_percentile": 0, "ScanBlockBytesScanned_95th_percentile": 0, "ScanBlockBytesScanned_98th_percentile": 0, "ScanBlockBytesScanned_99th_percentile": 0, "ScanBlockBytesScanned_99.9th_percentile": 0, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 0, "Delete_min": 0, "Delete_max": 0, "Delete_mean": 0, "Delete_25th_percentile": 0, "Delete_median": 0, "Delete_75th_percentile": 0, "Delete_90th_percentile": 0, "Delete_95th_percentile": 0, "Delete_98th_percentile": 0, "Delete_99th_percentile": 0, "Delete_99.9th_percentile": 0, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 0, "FlushMemstoreSize_num_ops": 0, "FlushMemstoreSize_min": 0, "FlushMemstoreSize_max": 0, "FlushMemstoreSize_mean": 0, "FlushMemstoreSize_25th_percentile": 0, "FlushMemstoreSize_median": 0, "FlushMemstoreSize_75th_percentile": 0, "FlushMemstoreSize_90th_percentile": 0, "FlushMemstoreSize_95th_percentile": 0, "FlushMemstoreSize_98th_percentile": 0, "FlushMemstoreSize_99th_percentile": 0, "FlushMemstoreSize_99.9th_percentile": 0, "CompactionInputFileCount_num_ops": 0, "CompactionInputFileCount_min": 0, "CompactionInputFileCount_max": 0, "CompactionInputFileCount_mean": 0, "CompactionInputFileCount_25th_percentile": 0, "CompactionInputFileCount_median": 0, "CompactionInputFileCount_75th_percentile": 0, "CompactionInputFileCount_90th_percentile": 0, "CompactionInputFileCount_95th_percentile": 0, "CompactionInputFileCount_98th_percentile": 0, "CompactionInputFileCount_99th_percentile": 0, "CompactionInputFileCount_99.9th_percentile": 0, "CompactionTime_num_ops": 0, "CompactionTime_min": 0, "CompactionTime_max": 0, "CompactionTime_mean": 0, "CompactionTime_25th_percentile": 0, "CompactionTime_median": 0, "CompactionTime_75th_percentile": 0, "CompactionTime_90th_percentile": 0, "CompactionTime_95th_percentile": 0, "CompactionTime_98th_percentile": 0, "CompactionTime_99th_percentile": 0, "CompactionTime_99.9th_percentile": 0, "Get_num_ops": 0, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "MajorCompactionInputFileCount_num_ops": 0, "MajorCompactionInputFileCount_min": 0, "MajorCompactionInputFileCount_max": 0, "MajorCompactionInputFileCount_mean": 0, "MajorCompactionInputFileCount_25th_percentile": 0, "MajorCompactionInputFileCount_median": 0, "MajorCompactionInputFileCount_75th_percentile": 0, "MajorCompactionInputFileCount_90th_percentile": 0, "MajorCompactionInputFileCount_95th_percentile": 0, "MajorCompactionInputFileCount_98th_percentile": 0, "MajorCompactionInputFileCount_99th_percentile": 0, "MajorCompactionInputFileCount_99.9th_percentile": 0, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 0, "MajorCompactionOutputSize_min": 0, "MajorCompactionOutputSize_max": 0, "MajorCompactionOutputSize_mean": 0, "MajorCompactionOutputSize_25th_percentile": 0, "MajorCompactionOutputSize_median": 0, "MajorCompactionOutputSize_75th_percentile": 0, "MajorCompactionOutputSize_90th_percentile": 0, "MajorCompactionOutputSize_95th_percentile": 0, "MajorCompactionOutputSize_98th_percentile": 0, "MajorCompactionOutputSize_99th_percentile": 0, "MajorCompactionOutputSize_99.9th_percentile": 0, "CompactionOutputFileCount_num_ops": 0, "CompactionOutputFileCount_min": 0, "CompactionOutputFileCount_max": 0, "CompactionOutputFileCount_mean": 0, "CompactionOutputFileCount_25th_percentile": 0, "CompactionOutputFileCount_median": 0, "CompactionOutputFileCount_75th_percentile": 0, "CompactionOutputFileCount_90th_percentile": 0, "CompactionOutputFileCount_95th_percentile": 0, "CompactionOutputFileCount_98th_percentile": 0, "CompactionOutputFileCount_99th_percentile": 0, "CompactionOutputFileCount_99.9th_percentile": 0, "slowDeleteCount": 0, "FlushTime_num_ops": 0, "FlushTime_min": 0, "FlushTime_max": 0, "FlushTime_mean": 0, "FlushTime_25th_percentile": 0, "FlushTime_median": 0, "FlushTime_75th_percentile": 0, "FlushTime_90th_percentile": 0, "FlushTime_95th_percentile": 0, "FlushTime_98th_percentile": 0, "FlushTime_99th_percentile": 0, "FlushTime_99.9th_percentile": 0, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 0, "MajorCompactionOutputFileCount_min": 0, "MajorCompactionOutputFileCount_max": 0, "MajorCompactionOutputFileCount_mean": 0, "MajorCompactionOutputFileCount_25th_percentile": 0, "MajorCompactionOutputFileCount_median": 0, "MajorCompactionOutputFileCount_75th_percentile": 0, "MajorCompactionOutputFileCount_90th_percentile": 0, "MajorCompactionOutputFileCount_95th_percentile": 0, "MajorCompactionOutputFileCount_98th_percentile": 0, "MajorCompactionOutputFileCount_99th_percentile": 0, "MajorCompactionOutputFileCount_99.9th_percentile": 0, "slowGetCount": 0, "ScanSize_num_ops": 0, "ScanSize_min": 0, "ScanSize_max": 0, "ScanSize_mean": 0, "ScanSize_25th_percentile": 0, "ScanSize_median": 0, "ScanSize_75th_percentile": 0, "ScanSize_90th_percentile": 0, "ScanSize_95th_percentile": 0, "ScanSize_98th_percentile": 0, "ScanSize_99th_percentile": 0, "ScanSize_99.9th_percentile": 0, "CompactionOutputSize_num_ops": 0, "CompactionOutputSize_min": 0, "CompactionOutputSize_max": 0, "CompactionOutputSize_mean": 0, "CompactionOutputSize_25th_percentile": 0, "CompactionOutputSize_median": 0, "CompactionOutputSize_75th_percentile": 0, "CompactionOutputSize_90th_percentile": 0, "CompactionOutputSize_95th_percentile": 0, "CompactionOutputSize_98th_percentile": 0, "CompactionOutputSize_99th_percentile": 0, "CompactionOutputSize_99.9th_percentile": 0, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-11-13T03:32:04,300 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41967 {}] master.MasterRpcServices(700): 8394b182524b,40261,1731468707776 reported a fatal error: ***** ABORTING region server 8394b182524b,40261,1731468707776: testing ***** 2024-11-13T03:32:04,302 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,40261,1731468707776' ***** 2024-11-13T03:32:04,302 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-11-13T03:32:04,302 INFO [RS:2;8394b182524b:40261 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:32:04,302 INFO [RS:2;8394b182524b:40261 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-11-13T03:32:04,302 INFO [RS:2;8394b182524b:40261 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-11-13T03:32:04,302 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(3091): Received CLOSE for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:04,303 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:32:04,303 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(956): aborting server 8394b182524b,40261,1731468707776 2024-11-13T03:32:04,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=43461 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:44054 deadline: 1731468784303, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=40261 startCode=1731468707776. As of locationSeqNum=12. 2024-11-13T03:32:04,303 INFO [RS:2;8394b182524b:40261 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:32:04,303 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:04,303 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,303 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,303 INFO [RS:2;8394b182524b:40261 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;8394b182524b:40261. 2024-11-13T03:32:04,303 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:04,303 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,303 DEBUG [RS:2;8394b182524b:40261 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:04,303 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,43461,1731468707627, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,43461,1731468707627, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=40261 startCode=1731468707776. As of locationSeqNum=12. 2024-11-13T03:32:04,304 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,43461,1731468707627, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=40261 startCode=1731468707776. As of locationSeqNum=12. 2024-11-13T03:32:04,304 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,43461,1731468707627, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=8394b182524b port=40261 startCode=1731468707776. As of locationSeqNum=12. 2024-11-13T03:32:04,304 DEBUG [RS:2;8394b182524b:40261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:04,304 INFO [RS:2;8394b182524b:40261 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:32:04,304 INFO [RS:2;8394b182524b:40261 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:32:04,304 INFO [RS:2;8394b182524b:40261 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:32:04,305 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T03:32:04,305 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:04,305 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,305 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468724303Running coprocessor pre-close hooks at 1731468724303Disabling compacts and flushes for region at 1731468724303Disabling writes for close at 1731468724303Writing region close event to WAL at 1731468724305 (+2 ms)Running coprocessor post-close hooks at 1731468724305Closed at 1731468724305 2024-11-13T03:32:04,306 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-11-13T03:32:04,306 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:04,306 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:32:04,306 DEBUG [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:32:04,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting 2024-11-13T03:32:04,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting 2024-11-13T03:32:04,306 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 from cache 2024-11-13T03:32:04,306 ERROR [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1960): Memstore data size is 5811 in region hbase:meta,,1.1588230740 2024-11-13T03:32:04,306 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:32:04,306 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:32:04,307 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731468724306Running coprocessor pre-close hooks at 1731468724306Disabling compacts and flushes for region at 1731468724306Disabling writes for close at 1731468724306Writing region close event to WAL at 1731468724306Running coprocessor post-close hooks at 1731468724306Closed at 1731468724306 2024-11-13T03:32:04,307 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T03:32:04,393 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:04,408 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:04,410 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1] 2024-11-13T03:32:04,410 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:04,411 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting 2024-11-13T03:32:04,411 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server 8394b182524b,40261,1731468707776 aborting 2024-11-13T03:32:04,412 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 from cache 2024-11-13T03:32:04,506 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,40261,1731468707776; all regions closed. 2024-11-13T03:32:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741836_1012 (size=6845) 2024-11-13T03:32:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741836_1012 (size=6845) 2024-11-13T03:32:04,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741836_1012 (size=6845) 2024-11-13T03:32:04,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741835_1011 (size=2142) 2024-11-13T03:32:04,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741835_1011 (size=2142) 2024-11-13T03:32:04,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741835_1011 (size=2142) 2024-11-13T03:32:04,521 DEBUG [RS:2;8394b182524b:40261 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:04,521 INFO [RS:2;8394b182524b:40261 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:04,521 INFO [RS:2;8394b182524b:40261 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:32:04,522 INFO [RS:2;8394b182524b:40261 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-13T03:32:04,522 INFO [RS:2;8394b182524b:40261 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:32:04,522 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:32:04,522 INFO [RS:2;8394b182524b:40261 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:40261 2024-11-13T03:32:04,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:32:04,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,40261,1731468707776 2024-11-13T03:32:04,611 INFO [RS:2;8394b182524b:40261 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:32:04,614 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,40261,1731468707776] 2024-11-13T03:32:04,618 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:04,620 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1] 2024-11-13T03:32:04,622 WARN [RPCClient-NioEventLoopGroup-6-3 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8394b182524b:40261 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:40261 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:04,622 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1, error=java.net.ConnectException: Call to address=8394b182524b:40261 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:40261 2024-11-13T03:32:04,622 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 is java.net.ConnectException: Connection refused 2024-11-13T03:32:04,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 from cache 2024-11-13T03:32:04,623 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.FailedServers(52): Added failed server with address 8394b182524b:40261 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: 8394b182524b/172.17.0.3:40261 2024-11-13T03:32:04,631 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,40261,1731468707776 already deleted, retry=false 2024-11-13T03:32:04,631 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of 8394b182524b,40261,1731468707776 on 8394b182524b,41967,1731468706868 2024-11-13T03:32:04,637 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure 8394b182524b,40261,1731468707776, splitWal=true, meta=true 2024-11-13T03:32:04,640 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1991): Scheduled ServerCrashProcedure pid=13 for 8394b182524b,40261,1731468707776 (carryingMeta=true) 8394b182524b,40261,1731468707776/CRASHED/regionCount=2/lock=java.util.concurrent.locks.ReentrantReadWriteLock@34098471[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-11-13T03:32:04,641 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure 8394b182524b,40261,1731468707776, splitWal=true, meta=true 2024-11-13T03:32:04,642 INFO [PEWorker-2 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_META_LOGS, hasLock=true; ServerCrashProcedure 8394b182524b,40261,1731468707776, splitWal=true, meta=true, isMeta: true 2024-11-13T03:32:04,644 DEBUG [PEWorker-2 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting 2024-11-13T03:32:04,646 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8394b182524b,40261,1731468707776 WAL count=1, meta=true 2024-11-13T03:32:04,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta}] 2024-11-13T03:32:04,655 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=8394b182524b,41763,1731468707723 2024-11-13T03:32:04,657 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, worker=8394b182524b,41763,1731468707723}] 2024-11-13T03:32:04,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:04,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40261-0x101323fbcd90003, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:04,722 INFO [RS:2;8394b182524b:40261 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:32:04,722 INFO [RS:2;8394b182524b:40261 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,40261,1731468707776; zookeeper connection closed. 2024-11-13T03:32:04,722 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@648aadc7 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@648aadc7 2024-11-13T03:32:04,820 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-13T03:32:04,823 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50897, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-13T03:32:04,825 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-11-13T03:32:04,844 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, size=6.7 K (6845bytes) 2024-11-13T03:32:04,844 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta 2024-11-13T03:32:04,845 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta after 1ms 2024-11-13T03:32:04,847 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta took 3ms 2024-11-13T03:32:04,854 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 1588230740: last_flushed_sequence_id: 18446744073709551615 store_sequence_id { family_name: "info" sequence_id: 5 } store_sequence_id { family_name: "ns" sequence_id: 3 } store_sequence_id { family_name: "rep_barrier" sequence_id: 18446744073709551615 } store_sequence_id { family_name: "table" sequence_id: 6 } 2024-11-13T03:32:04,854 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta so closing down 2024-11-13T03:32:04,855 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:04,856 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp 2024-11-13T03:32:04,857 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp 2024-11-13T03:32:04,858 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:04,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741892_1070 (size=6604) 2024-11-13T03:32:04,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741892_1070 (size=6604) 2024-11-13T03:32:04,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741892_1070 (size=6604) 2024-11-13T03:32:04,867 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:04,868 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T03:32:04,868 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 16 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false 2024-11-13T03:32:04,868 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, journal: Splitting hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, size=6.7 K (6845bytes) at 1731468724844Finishing writing output for hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta so closing down at 1731468724855 (+11 ms)Creating recovered edits writer path=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp at 1731468724857 (+2 ms)3 split writer threads finished at 1731468724858 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1731468724867 (+9 ms)Rename recovered edits hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000004-8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta.temp to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 at 1731468724868 (+1 ms)Processed 16 edits across 1 Regions in 20 ms; skipped=1; WAL=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, size=6.7 K, length=6845, corrupted=false, cancelled=false at 1731468724868 2024-11-13T03:32:04,869 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta 2024-11-13T03:32:04,870 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-11-13T03:32:04,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-11-13T03:32:04,877 INFO [PEWorker-4 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs 2024-11-13T03:32:04,879 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-11-13T03:32:04,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, worker=8394b182524b,41763,1731468707723 in 220 msec 2024-11-13T03:32:04,881 DEBUG [PEWorker-5 {}] master.SplitWALManager(172): Release split WAL worker=8394b182524b,41763,1731468707723 2024-11-13T03:32:04,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-13T03:32:04,885 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 8394b182524b%2C40261%2C1731468707776.meta.1731468709826.meta, worker=8394b182524b,41763,1731468707723 in 234 msec 2024-11-13T03:32:04,887 INFO [PEWorker-2 {}] master.SplitLogManager(171): hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting dir is empty, no logs to split. 2024-11-13T03:32:04,887 INFO [PEWorker-2 {}] master.SplitWALManager(105): 8394b182524b,40261,1731468707776 WAL count=0, meta=true 2024-11-13T03:32:04,887 DEBUG [PEWorker-2 {}] procedure.ServerCrashProcedure(329): Check if 8394b182524b,40261,1731468707776 WAL splitting is done? wals=0, meta=true 2024-11-13T03:32:04,889 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-13T03:32:04,890 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:32:04,891 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T03:32:04,928 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:04,929 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1] 2024-11-13T03:32:04,929 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8394b182524b:40261 this server is in the failed servers list 2024-11-13T03:32:04,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8394b182524b:40261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:04,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:04,930 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 from cache 2024-11-13T03:32:05,041 DEBUG [8394b182524b:41967 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=2, allServersCount=2 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:05,042 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:05,042 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:05,042 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:05,042 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=8394b182524b,43461,1731468707627 2024-11-13T03:32:05,043 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,43461,1731468707627, state=OPENING 2024-11-13T03:32:05,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,094 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,094 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-13T03:32:05,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,094 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,094 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,43461,1731468707627}] 2024-11-13T03:32:05,258 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-13T03:32:05,259 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:05,259 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-13T03:32:05,262 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=8394b182524b%2C43461%2C1731468707627.meta, suffix=.meta, logDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627, archiveDir=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs, maxLogs=32 2024-11-13T03:32:05,275 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627/8394b182524b%2C43461%2C1731468707627.meta.1731468725262.meta, exclude list is [], retry=0 2024-11-13T03:32:05,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:05,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:05,277 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:05,279 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627/8394b182524b%2C43461%2C1731468707627.meta.1731468725262.meta 2024-11-13T03:32:05,280 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:32:05,280 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:05,280 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-13T03:32:05,280 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-13T03:32:05,281 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-13T03:32:05,281 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-13T03:32:05,281 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:05,281 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-13T03:32:05,281 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-13T03:32:05,283 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-13T03:32:05,284 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-13T03:32:05,284 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:05,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:05,285 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-13T03:32:05,285 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-13T03:32:05,285 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:05,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:05,286 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-13T03:32:05,287 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-13T03:32:05,287 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:05,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:05,287 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-13T03:32:05,288 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-13T03:32:05,288 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:05,288 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-13T03:32:05,288 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-13T03:32:05,289 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:32:05,290 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740 2024-11-13T03:32:05,291 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T03:32:05,294 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5793): Applied 40, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=18, path=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T03:32:05,295 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=5.67 KB heapSize=9.66 KB 2024-11-13T03:32:05,314 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/06d8eb96c2234741bfc9606f4c952f9c is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270./info:regioninfo/1731468724223/Put/seqid=0 2024-11-13T03:32:05,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741894_1072 (size=11177) 2024-11-13T03:32:05,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741894_1072 (size=11177) 2024-11-13T03:32:05,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741894_1072 (size=11177) 2024-11-13T03:32:05,320 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=5.46 KB at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/06d8eb96c2234741bfc9606f4c952f9c 2024-11-13T03:32:05,339 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/ns/60c6f6b0c6d1440294a1f750d5e2b2a8 is 43, key is default/ns:d/1731468710063/Put/seqid=0 2024-11-13T03:32:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741895_1073 (size=5153) 2024-11-13T03:32:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741895_1073 (size=5153) 2024-11-13T03:32:05,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741895_1073 (size=5153) 2024-11-13T03:32:05,346 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/ns/60c6f6b0c6d1440294a1f750d5e2b2a8 2024-11-13T03:32:05,364 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/table/a95c3d342c0942329dbaffcb510e3ae7 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1731468722424/Put/seqid=0 2024-11-13T03:32:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741896_1074 (size=5431) 2024-11-13T03:32:05,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741896_1074 (size=5431) 2024-11-13T03:32:05,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741896_1074 (size=5431) 2024-11-13T03:32:05,370 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=18 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/table/a95c3d342c0942329dbaffcb510e3ae7 2024-11-13T03:32:05,376 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/06d8eb96c2234741bfc9606f4c952f9c as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/info/06d8eb96c2234741bfc9606f4c952f9c 2024-11-13T03:32:05,382 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/info/06d8eb96c2234741bfc9606f4c952f9c, entries=36, sequenceid=18, filesize=10.9 K 2024-11-13T03:32:05,384 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/ns/60c6f6b0c6d1440294a1f750d5e2b2a8 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/ns/60c6f6b0c6d1440294a1f750d5e2b2a8 2024-11-13T03:32:05,389 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/ns/60c6f6b0c6d1440294a1f750d5e2b2a8, entries=2, sequenceid=18, filesize=5.0 K 2024-11-13T03:32:05,390 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/table/a95c3d342c0942329dbaffcb510e3ae7 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/table/a95c3d342c0942329dbaffcb510e3ae7 2024-11-13T03:32:05,396 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/table/a95c3d342c0942329dbaffcb510e3ae7, entries=2, sequenceid=18, filesize=5.3 K 2024-11-13T03:32:05,396 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(3140): Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=18, compaction requested=false; wal=null 2024-11-13T03:32:05,397 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/0000000000000000018 2024-11-13T03:32:05,398 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-13T03:32:05,398 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-13T03:32:05,399 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-13T03:32:05,400 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-13T03:32:05,402 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/18.seqid, newMaxSeqId=18, maxSeqId=1 2024-11-13T03:32:05,403 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=19; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=75385325, jitterRate=0.12332887947559357}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-13T03:32:05,403 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-13T03:32:05,404 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731468725281Writing region info on filesystem at 1731468725281Initializing all the Stores at 1731468725282 (+1 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468725282Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468725283 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468725283Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731468725283Obtaining lock to block concurrent updates at 1731468725295 (+12 ms)Preparing flush snapshotting stores in 1588230740 at 1731468725295Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=5811, getHeapSize=9832, getOffHeapSize=0, getCellsCount=40 at 1731468725295Flushing stores of hbase:meta,,1.1588230740 at 1731468725295Flushing 1588230740/info: creating writer at 1731468725295Flushing 1588230740/info: appending metadata at 1731468725313 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731468725314 (+1 ms)Flushing 1588230740/ns: creating writer at 1731468725326 (+12 ms)Flushing 1588230740/ns: appending metadata at 1731468725339 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731468725339Flushing 1588230740/table: creating writer at 1731468725351 (+12 ms)Flushing 1588230740/table: appending metadata at 1731468725363 (+12 ms)Flushing 1588230740/table: closing flushed file at 1731468725364 (+1 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e2fb2be: reopening flushed file at 1731468725375 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a61f7b4: reopening flushed file at 1731468725383 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6da1aeea: reopening flushed file at 1731468725389 (+6 ms)Finished flush of dataSize ~5.67 KB/5811, heapSize ~9.37 KB/9592, currentSize=0 B/0 for 1588230740 in 102ms, sequenceid=18, compaction requested=false; wal=null at 1731468725396 (+7 ms)Cleaning up temporary data from old regions at 1731468725398 (+2 ms)Running coprocessor post-open hooks at 1731468725403 (+5 ms)Region opened successfully at 1731468725404 (+1 ms) 2024-11-13T03:32:05,405 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=17, masterSystemTime=1731468725249 2024-11-13T03:32:05,407 DEBUG [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-13T03:32:05,407 INFO [RS_OPEN_META-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_META, pid=17}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-13T03:32:05,408 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=19, regionLocation=8394b182524b,43461,1731468707627 2024-11-13T03:32:05,409 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 8394b182524b,43461,1731468707627, state=OPEN 2024-11-13T03:32:05,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,420 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-13T03:32:05,420 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=17, ppid=16, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=8394b182524b,43461,1731468707627 2024-11-13T03:32:05,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,420 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-13T03:32:05,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-11-13T03:32:05,423 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=8394b182524b,43461,1731468707627 in 326 msec 2024-11-13T03:32:05,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-11-13T03:32:05,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 534 msec 2024-11-13T03:32:05,425 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(207): 8394b182524b,40261,1731468707776 had 2 regions 2024-11-13T03:32:05,426 INFO [PEWorker-1 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure 8394b182524b,40261,1731468707776, splitWal=true, meta=true, isMeta: false 2024-11-13T03:32:05,428 INFO [PEWorker-1 {}] master.SplitWALManager(105): 8394b182524b,40261,1731468707776 WAL count=1, meta=false 2024-11-13T03:32:05,428 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure 8394b182524b%2C40261%2C1731468707776.1731468709326}] 2024-11-13T03:32:05,430 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=8394b182524b,41763,1731468707723 2024-11-13T03:32:05,431 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=19, ppid=18, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C40261%2C1731468707776.1731468709326, worker=8394b182524b,41763,1731468707723}] 2024-11-13T03:32:05,437 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:05,439 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,43461,1731468707627, seqNum=-1] 2024-11-13T03:32:05,558 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8394b182524b:40261 this server is in the failed servers list 2024-11-13T03:32:05,558 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=12, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8394b182524b:40261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:05,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=12 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:05,559 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=12 from cache 2024-11-13T03:32:05,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41763 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=19 2024-11-13T03:32:05,608 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326, size=2.1 K (2142bytes) 2024-11-13T03:32:05,608 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 2024-11-13T03:32:05,608 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 after 0ms 2024-11-13T03:32:05,611 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(310): Open hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 took 3ms 2024-11-13T03:32:05,613 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(352): Last flushed sequenceid for fa8df6bead70f7386f753fb4aa9c8270: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-11-13T03:32:05,614 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 so closing down 2024-11-13T03:32:05,614 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:05,614 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:05,614 INFO [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326, size=2.1 K, length=2142, corrupted=false, cancelled=false 2024-11-13T03:32:05,614 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326, journal: Splitting hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326, size=2.1 K (2142bytes) at 1731468725608Finishing writing output for hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 so closing down at 1731468725614 (+6 ms)3 split writer threads finished at 1731468725614Processed 6 edits across 0 Regions in 3 ms; skipped=6; WAL=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326, size=2.1 K, length=2142, corrupted=false, cancelled=false at 1731468725614 2024-11-13T03:32:05,614 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 2024-11-13T03:32:05,614 DEBUG [RS_LOG_REPLAY_OPS-regionserver/8394b182524b:0-1 {event_type=RS_LOG_REPLAY, pid=19}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=19 2024-11-13T03:32:05,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=41967 {}] master.HMaster(4169): Remote procedure done, pid=19 2024-11-13T03:32:05,618 INFO [PEWorker-5 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting/8394b182524b%2C40261%2C1731468707776.1731468709326 to hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs 2024-11-13T03:32:05,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=19, resume processing ppid=18 2024-11-13T03:32:05,621 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, ppid=18, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure 8394b182524b%2C40261%2C1731468707776.1731468709326, worker=8394b182524b,41763,1731468707723 in 187 msec 2024-11-13T03:32:05,622 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=8394b182524b,41763,1731468707723 2024-11-13T03:32:05,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=13 2024-11-13T03:32:05,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure 8394b182524b%2C40261%2C1731468707776.1731468709326, worker=8394b182524b,41763,1731468707723 in 194 msec 2024-11-13T03:32:05,626 INFO [PEWorker-3 {}] master.SplitLogManager(171): hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,40261,1731468707776-splitting dir is empty, no logs to split. 2024-11-13T03:32:05,626 INFO [PEWorker-3 {}] master.SplitWALManager(105): 8394b182524b,40261,1731468707776 WAL count=0, meta=false 2024-11-13T03:32:05,626 DEBUG [PEWorker-3 {}] procedure.ServerCrashProcedure(329): Check if 8394b182524b,40261,1731468707776 WAL splitting is done? wals=0, meta=false 2024-11-13T03:32:05,628 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN}] 2024-11-13T03:32:05,630 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN 2024-11-13T03:32:05,631 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-11-13T03:32:05,777 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18] 2024-11-13T03:32:05,777 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8394b182524b:40261 this server is in the failed servers list 2024-11-13T03:32:05,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8394b182524b:40261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:05,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:05,778 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 from cache 2024-11-13T03:32:05,781 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(204): Hosts are {8394b182524b=0} racks are {/default-rack=0} 2024-11-13T03:32:05,781 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-11-13T03:32:05,781 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-11-13T03:32:05,782 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-11-13T03:32:05,782 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-11-13T03:32:05,782 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-11-13T03:32:05,782 INFO [8394b182524b:41967 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-11-13T03:32:05,782 DEBUG [8394b182524b:41967 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-11-13T03:32:05,782 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPENING, regionLocation=8394b182524b,41763,1731468707723 2024-11-13T03:32:05,783 WARN [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server 8394b182524b:40261 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8394b182524b/172.17.0.3:40261 Caused by: java.net.ConnectException: finishConnect(..) failed: Connection refused at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.newConnectException0(Errors.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Errors.handleConnectErrno(Errors.java:131) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.unix.Socket.finishConnect(Socket.java:359) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.doFinishConnect(AbstractEpollChannel.java:715) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:692) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:05,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 , the old value is region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1, error=java.net.ConnectException: Call to address=8394b182524b:40261 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8394b182524b/172.17.0.3:40261 2024-11-13T03:32:05,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 is java.net.ConnectException: finishConnect(..) failed: Connection refused 2024-11-13T03:32:05,784 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] client.AsyncRegionLocatorHelper(88): Try removing region=hbase:meta,,1.1588230740, hostname=8394b182524b,40261,1731468707776, seqNum=-1 from cache 2024-11-13T03:32:05,785 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.FailedServers(52): Added failed server with address 8394b182524b:40261 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: finishConnect(..) failed: Connection refused: 8394b182524b/172.17.0.3:40261 2024-11-13T03:32:05,888 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-13T03:32:05,888 DEBUG [Async-Client-Retry-Timer-pool-0 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=8394b182524b,43461,1731468707627, seqNum=-1] 2024-11-13T03:32:05,889 DEBUG [Async-Client-Retry-Timer-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:05,890 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44605, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:05,912 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=20, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN because future has completed 2024-11-13T03:32:05,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,41763,1731468707723}] 2024-11-13T03:32:06,073 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:06,073 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7752): Opening region: {ENCODED => fa8df6bead70f7386f753fb4aa9c8270, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:06,073 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,074 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:06,074 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7794): checking encryption for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,074 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(7797): checking classloading for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,076 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,077 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf1 2024-11-13T03:32:06,077 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:06,084 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf1/c59df0d954784ea2b48247cef05c723c 2024-11-13T03:32:06,085 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:06,085 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,086 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region fa8df6bead70f7386f753fb4aa9c8270 columnFamilyName cf2 2024-11-13T03:32:06,086 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:06,092 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18] 2024-11-13T03:32:06,092 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.AbstractRpcClient(357): Not trying to connect to 8394b182524b:40261 this server is in the failed servers list 2024-11-13T03:32:06,093 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=8394b182524b:40261 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:06,093 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: 8394b182524b:40261 2024-11-13T03:32:06,093 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,40261,1731468707776, seqNum=18 from cache 2024-11-13T03:32:06,093 DEBUG [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/cf2/dc04e3dce7c448c585484514a61bc486 2024-11-13T03:32:06,093 INFO [StoreOpener-fa8df6bead70f7386f753fb4aa9c8270-1 {}] regionserver.HStore(327): Store=fa8df6bead70f7386f753fb4aa9c8270/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:06,094 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1038): replaying wal for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,095 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,096 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,097 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1048): stopping wal replay for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,097 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1060): Cleaning up temporary data for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,097 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-11-13T03:32:06,099 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1093): writing seq id for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,100 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1114): Opened fa8df6bead70f7386f753fb4aa9c8270; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72374798, jitterRate=0.07846853137016296}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-11-13T03:32:06,100 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1122): Running coprocessor post-open hooks for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:06,100 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegion(1006): Region open journal for fa8df6bead70f7386f753fb4aa9c8270: Running coprocessor pre-open hook at 1731468726074Writing region info on filesystem at 1731468726074Initializing all the Stores at 1731468726075 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468726075Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468726075Cleaning up temporary data from old regions at 1731468726097 (+22 ms)Running coprocessor post-open hooks at 1731468726100 (+3 ms)Region opened successfully at 1731468726100 2024-11-13T03:32:06,102 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., pid=21, masterSystemTime=1731468726065 2024-11-13T03:32:06,104 DEBUG [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:06,104 INFO [RS_OPEN_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_OPEN_REGION, pid=21}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:06,105 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=20 updating hbase:meta row=fa8df6bead70f7386f753fb4aa9c8270, regionState=OPEN, openSeqNum=18, regionLocation=8394b182524b,41763,1731468707723 2024-11-13T03:32:06,107 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=21, ppid=20, state=RUNNABLE, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,41763,1731468707723 because future has completed 2024-11-13T03:32:06,110 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=21, resume processing ppid=20 2024-11-13T03:32:06,111 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, ppid=20, state=SUCCESS, hasLock=false; OpenRegionProcedure fa8df6bead70f7386f753fb4aa9c8270, server=8394b182524b,41763,1731468707723 in 195 msec 2024-11-13T03:32:06,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=13 2024-11-13T03:32:06,113 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(291): removed crashed server 8394b182524b,40261,1731468707776 after splitting done 2024-11-13T03:32:06,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=fa8df6bead70f7386f753fb4aa9c8270, ASSIGN in 483 msec 2024-11-13T03:32:06,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure 8394b182524b,40261,1731468707776, splitWal=true, meta=true in 1.4810 sec 2024-11-13T03:32:06,602 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270., hostname=8394b182524b,41763,1731468707723, seqNum=18] 2024-11-13T03:32:06,603 DEBUG [RPCClient-NioEventLoopGroup-6-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-13T03:32:06,605 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-4-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48306, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-13T03:32:06,617 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=410 (was 413), OpenFileDescriptor=1050 (was 1025) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 316), ProcessCount=11 (was 11), AvailableMemoryMB=2914 (was 2965) 2024-11-13T03:32:06,617 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-13T03:32:06,629 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=409, OpenFileDescriptor=1050, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=2912 2024-11-13T03:32:06,629 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1050 is superior to 1024 2024-11-13T03:32:06,643 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:06,645 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:06,645 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:06,648 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-73006374, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-73006374, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:06,660 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-73006374/hregion-73006374.1731468726648, exclude list is [], retry=0 2024-11-13T03:32:06,662 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:06,663 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:06,663 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:06,664 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-73006374/hregion-73006374.1731468726648 2024-11-13T03:32:06,665 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:06,665 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f4adacacbdebe9176523c535885eccda, NAME => 'testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:32:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741898_1076 (size=67) 2024-11-13T03:32:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741898_1076 (size=67) 2024-11-13T03:32:06,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741898_1076 (size=67) 2024-11-13T03:32:07,077 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:07,080 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,084 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName a 2024-11-13T03:32:07,084 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,086 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,086 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,089 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName b 2024-11-13T03:32:07,089 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,090 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,090 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,091 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName c 2024-11-13T03:32:07,091 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,092 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,092 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,093 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,095 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,095 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,095 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:07,097 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,099 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:07,100 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4adacacbdebe9176523c535885eccda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68852295, jitterRate=0.025979146361351013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4adacacbdebe9176523c535885eccda: Writing region info on filesystem at 1731468727077Initializing all the Stores at 1731468727079 (+2 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727079Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727080 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727080Cleaning up temporary data from old regions at 1731468727095 (+15 ms)Region opened successfully at 1731468727101 (+6 ms) 2024-11-13T03:32:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f4adacacbdebe9176523c535885eccda, disabling compactions & flushes 2024-11-13T03:32:07,101 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,101 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,102 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. after waiting 0 ms 2024-11-13T03:32:07,102 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,102 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,102 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f4adacacbdebe9176523c535885eccda: Waiting for close lock at 1731468727101Disabling compacts and flushes for region at 1731468727101Disabling writes for close at 1731468727102 (+1 ms)Writing region close event to WAL at 1731468727102Closed at 1731468727102 2024-11-13T03:32:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741897_1075 (size=93) 2024-11-13T03:32:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741897_1075 (size=93) 2024-11-13T03:32:07,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741897_1075 (size=93) 2024-11-13T03:32:07,108 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:07,108 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-73006374:(num 1731468726648) 2024-11-13T03:32:07,108 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:07,110 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:07,123 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, exclude list is [], retry=0 2024-11-13T03:32:07,125 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:07,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:07,126 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:07,127 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 2024-11-13T03:32:07,128 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:07,128 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f4adacacbdebe9176523c535885eccda, NAME => 'testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:07,128 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:07,128 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,128 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,130 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,130 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName a 2024-11-13T03:32:07,131 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,131 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,131 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,132 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName b 2024-11-13T03:32:07,132 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,132 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,132 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,133 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName c 2024-11-13T03:32:07,133 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,133 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,134 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,134 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,135 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,136 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,136 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,137 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:07,138 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,139 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4adacacbdebe9176523c535885eccda; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74965424, jitterRate=0.11707186698913574}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:07,139 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4adacacbdebe9176523c535885eccda: Writing region info on filesystem at 1731468727128Initializing all the Stores at 1731468727129 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727129Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727129Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727129Cleaning up temporary data from old regions at 1731468727136 (+7 ms)Region opened successfully at 1731468727139 (+3 ms) 2024-11-13T03:32:07,160 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4adacacbdebe9176523c535885eccda 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T03:32:07,181 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/a/88a0a3cd7f734e7a9efec6c75adb5e1a is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731468727139/Put/seqid=0 2024-11-13T03:32:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:07,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741900_1078 (size=5958) 2024-11-13T03:32:07,187 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/a/88a0a3cd7f734e7a9efec6c75adb5e1a 2024-11-13T03:32:07,206 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/cdabbd13374f47b6a7e6a7c3e8251550 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468727147/Put/seqid=0 2024-11-13T03:32:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741901_1079 (size=5958) 2024-11-13T03:32:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741901_1079 (size=5958) 2024-11-13T03:32:07,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741901_1079 (size=5958) 2024-11-13T03:32:07,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-11-13T03:32:07,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:07,215 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/cdabbd13374f47b6a7e6a7c3e8251550 2024-11-13T03:32:07,215 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-11-13T03:32:07,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:07,217 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T03:32:07,217 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:07,218 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-11-13T03:32:07,218 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:07,239 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/c/8eb2989619ae426591ea7b9c74283111 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731468727153/Put/seqid=0 2024-11-13T03:32:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741902_1080 (size=5958) 2024-11-13T03:32:07,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741902_1080 (size=5958) 2024-11-13T03:32:07,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741902_1080 (size=5958) 2024-11-13T03:32:07,246 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/c/8eb2989619ae426591ea7b9c74283111 2024-11-13T03:32:07,253 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/a/88a0a3cd7f734e7a9efec6c75adb5e1a as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/a/88a0a3cd7f734e7a9efec6c75adb5e1a 2024-11-13T03:32:07,259 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/a/88a0a3cd7f734e7a9efec6c75adb5e1a, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:07,261 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/cdabbd13374f47b6a7e6a7c3e8251550 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/cdabbd13374f47b6a7e6a7c3e8251550 2024-11-13T03:32:07,267 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/cdabbd13374f47b6a7e6a7c3e8251550, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:07,268 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/c/8eb2989619ae426591ea7b9c74283111 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/c/8eb2989619ae426591ea7b9c74283111 2024-11-13T03:32:07,275 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/c/8eb2989619ae426591ea7b9c74283111, entries=10, sequenceid=33, filesize=5.8 K 2024-11-13T03:32:07,276 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f4adacacbdebe9176523c535885eccda in 116ms, sequenceid=33, compaction requested=false 2024-11-13T03:32:07,276 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f4adacacbdebe9176523c535885eccda: 2024-11-13T03:32:07,276 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f4adacacbdebe9176523c535885eccda, disabling compactions & flushes 2024-11-13T03:32:07,276 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,277 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,277 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. after waiting 0 ms 2024-11-13T03:32:07,277 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,278 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. 2024-11-13T03:32:07,278 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f4adacacbdebe9176523c535885eccda: Waiting for close lock at 1731468727276Disabling compacts and flushes for region at 1731468727276Disabling writes for close at 1731468727277 (+1 ms)Writing region close event to WAL at 1731468727278 (+1 ms)Closed at 1731468727278 2024-11-13T03:32:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741899_1077 (size=5937) 2024-11-13T03:32:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741899_1077 (size=5937) 2024-11-13T03:32:07,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741899_1077 (size=5937) 2024-11-13T03:32:07,288 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/cdabbd13374f47b6a7e6a7c3e8251550 to hdfs://localhost:33705/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/cdabbd13374f47b6a7e6a7c3e8251550 2024-11-13T03:32:07,304 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, size=5.8 K (5937bytes) 2024-11-13T03:32:07,304 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 2024-11-13T03:32:07,304 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 after 0ms 2024-11-13T03:32:07,307 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 took 4ms 2024-11-13T03:32:07,310 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 so closing down 2024-11-13T03:32:07,310 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:07,310 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468727110.temp 2024-11-13T03:32:07,312 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp 2024-11-13T03:32:07,312 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:07,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741903_1081 (size=5223) 2024-11-13T03:32:07,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741903_1081 (size=5223) 2024-11-13T03:32:07,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741903_1081 (size=5223) 2024-11-13T03:32:07,320 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:07,322 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 2024-11-13T03:32:07,322 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, size=5.8 K, length=5937, corrupted=false, cancelled=false 2024-11-13T03:32:07,322 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, journal: Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, size=5.8 K (5937bytes) at 1731468727304Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 so closing down at 1731468727310 (+6 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp at 1731468727312 (+2 ms)3 split writer threads finished at 1731468727312Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468727320 (+8 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000003-wal.1731468727110.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 at 1731468727322 (+2 ms)Processed 32 edits across 1 Regions in 14 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110, size=5.8 K, length=5937, corrupted=false, cancelled=false at 1731468727322 2024-11-13T03:32:07,324 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727110 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468727110 2024-11-13T03:32:07,325 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 2024-11-13T03:32:07,325 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:07,327 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:07,346 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727328, exclude list is [], retry=0 2024-11-13T03:32:07,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:07,350 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:07,351 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:07,352 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727328 2024-11-13T03:32:07,353 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:07,353 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f4adacacbdebe9176523c535885eccda, NAME => 'testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:07,353 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:07,353 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,353 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,355 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,356 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName a 2024-11-13T03:32:07,356 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,361 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/a/88a0a3cd7f734e7a9efec6c75adb5e1a 2024-11-13T03:32:07,362 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,362 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,363 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName b 2024-11-13T03:32:07,363 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,364 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,364 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,365 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f4adacacbdebe9176523c535885eccda columnFamilyName c 2024-11-13T03:32:07,365 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,373 DEBUG [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/c/8eb2989619ae426591ea7b9c74283111 2024-11-13T03:32:07,373 INFO [StoreOpener-f4adacacbdebe9176523c535885eccda-1 {}] regionserver.HStore(327): Store=f4adacacbdebe9176523c535885eccda/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,373 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,374 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,376 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,376 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 2024-11-13T03:32:07,380 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 2024-11-13T03:32:07,380 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f4adacacbdebe9176523c535885eccda 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T03:32:07,395 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/3b1727018bb648519ebc353b2c35bdf6 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468727147/Put/seqid=0 2024-11-13T03:32:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741905_1083 (size=5958) 2024-11-13T03:32:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741905_1083 (size=5958) 2024-11-13T03:32:07,406 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741905_1083 (size=5958) 2024-11-13T03:32:07,407 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/3b1727018bb648519ebc353b2c35bdf6 2024-11-13T03:32:07,415 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/.tmp/b/3b1727018bb648519ebc353b2c35bdf6 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/3b1727018bb648519ebc353b2c35bdf6 2024-11-13T03:32:07,423 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/b/3b1727018bb648519ebc353b2c35bdf6, entries=10, sequenceid=32, filesize=5.8 K 2024-11-13T03:32:07,423 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f4adacacbdebe9176523c535885eccda in 43ms, sequenceid=32, compaction requested=false; wal=null 2024-11-13T03:32:07,424 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/0000000000000000032 2024-11-13T03:32:07,426 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,426 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,427 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:07,428 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f4adacacbdebe9176523c535885eccda 2024-11-13T03:32:07,431 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/f4adacacbdebe9176523c535885eccda/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-11-13T03:32:07,432 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f4adacacbdebe9176523c535885eccda; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66232631, jitterRate=-0.013056889176368713}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:07,433 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f4adacacbdebe9176523c535885eccda: Writing region info on filesystem at 1731468727353Initializing all the Stores at 1731468727354 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727354Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727355 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727355Obtaining lock to block concurrent updates at 1731468727380 (+25 ms)Preparing flush snapshotting stores in f4adacacbdebe9176523c535885eccda at 1731468727380Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1731468727380Flushing stores of testReplayEditsWrittenViaHRegion,,1731468726644.f4adacacbdebe9176523c535885eccda. at 1731468727380Flushing f4adacacbdebe9176523c535885eccda/b: creating writer at 1731468727380Flushing f4adacacbdebe9176523c535885eccda/b: appending metadata at 1731468727395 (+15 ms)Flushing f4adacacbdebe9176523c535885eccda/b: closing flushed file at 1731468727395Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@547b9d68: reopening flushed file at 1731468727414 (+19 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f4adacacbdebe9176523c535885eccda in 43ms, sequenceid=32, compaction requested=false; wal=null at 1731468727423 (+9 ms)Cleaning up temporary data from old regions at 1731468727426 (+3 ms)Region opened successfully at 1731468727433 (+7 ms) 2024-11-13T03:32:07,458 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=418 (was 409) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:54020 [Waiting for operation #47] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:51160 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:58996 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:54180 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:40416 [Waiting for operation #44] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:59208 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741904_1082, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1120 (was 1050) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=299 (was 299), ProcessCount=11 (was 11), AvailableMemoryMB=2883 (was 2912) 2024-11-13T03:32:07,459 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1120 is superior to 1024 2024-11-13T03:32:07,472 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=418, OpenFileDescriptor=1120, MaxFileDescriptor=1048576, SystemLoadAverage=299, ProcessCount=11, AvailableMemoryMB=2882 2024-11-13T03:32:07,472 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1120 is superior to 1024 2024-11-13T03:32:07,494 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:07,497 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:07,498 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:07,500 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-67618646, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-67618646, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:07,514 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-67618646/hregion-67618646.1731468727501, exclude list is [], retry=0 2024-11-13T03:32:07,517 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:07,518 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:07,518 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:07,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-67618646/hregion-67618646.1731468727501 2024-11-13T03:32:07,522 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:07,522 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 756be66b118f1073fd36299191edec1f, NAME => 'testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:32:07,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741907_1085 (size=68) 2024-11-13T03:32:07,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741907_1085 (size=68) 2024-11-13T03:32:07,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741907_1085 (size=68) 2024-11-13T03:32:07,539 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:07,540 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,542 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName a 2024-11-13T03:32:07,542 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,543 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,543 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,545 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName b 2024-11-13T03:32:07,545 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,546 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,546 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,548 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName c 2024-11-13T03:32:07,548 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,548 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,549 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,550 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,550 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,551 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,552 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,552 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:07,553 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,556 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:07,556 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 756be66b118f1073fd36299191edec1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61454893, jitterRate=-0.08425073325634003}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 756be66b118f1073fd36299191edec1f: Writing region info on filesystem at 1731468727539Initializing all the Stores at 1731468727540 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727540Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727540Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727540Cleaning up temporary data from old regions at 1731468727552 (+12 ms)Region opened successfully at 1731468727557 (+5 ms) 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 756be66b118f1073fd36299191edec1f, disabling compactions & flushes 2024-11-13T03:32:07,557 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. after waiting 0 ms 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:07,557 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:07,557 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 756be66b118f1073fd36299191edec1f: Waiting for close lock at 1731468727557Disabling compacts and flushes for region at 1731468727557Disabling writes for close at 1731468727557Writing region close event to WAL at 1731468727557Closed at 1731468727557 2024-11-13T03:32:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741906_1084 (size=93) 2024-11-13T03:32:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741906_1084 (size=93) 2024-11-13T03:32:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741906_1084 (size=93) 2024-11-13T03:32:07,563 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:07,563 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-67618646:(num 1731468727501) 2024-11-13T03:32:07,563 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:07,565 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:07,579 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, exclude list is [], retry=0 2024-11-13T03:32:07,583 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:07,583 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:07,584 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:07,585 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 2024-11-13T03:32:07,586 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:32:07,643 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 756be66b118f1073fd36299191edec1f, NAME => 'testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:07,645 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,645 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:07,645 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,646 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,648 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,649 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName a 2024-11-13T03:32:07,649 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,650 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,650 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,651 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName b 2024-11-13T03:32:07,651 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,651 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,651 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,652 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName c 2024-11-13T03:32:07,652 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:07,653 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:07,653 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,653 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,655 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,656 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,656 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,656 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:07,658 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,659 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 756be66b118f1073fd36299191edec1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69033924, jitterRate=0.02868562936782837}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:07,659 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:07,659 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 756be66b118f1073fd36299191edec1f: Running coprocessor pre-open hook at 1731468727646Writing region info on filesystem at 1731468727646Initializing all the Stores at 1731468727647 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727647Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727647Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468727647Cleaning up temporary data from old regions at 1731468727656 (+9 ms)Running coprocessor post-open hooks at 1731468727659 (+3 ms)Region opened successfully at 1731468727659 2024-11-13T03:32:07,671 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 756be66b118f1073fd36299191edec1f 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-11-13T03:32:07,672 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:07,956 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:32:08,673 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:09,674 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:10,676 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:11,677 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:12,678 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:13,150 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:32:13,680 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:14,681 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:15,682 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:16,023 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-13T03:32:16,683 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for 756be66b118f1073fd36299191edec1f/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:16,684 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 756be66b118f1073fd36299191edec1f: 2024-11-13T03:32:16,684 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,695 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 756be66b118f1073fd36299191edec1f: 2024-11-13T03:32:16,695 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-11-13T03:32:16,695 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 756be66b118f1073fd36299191edec1f, disabling compactions & flushes 2024-11-13T03:32:16,695 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,695 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,695 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. after waiting 0 ms 2024-11-13T03:32:16,695 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,695 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,695 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. 2024-11-13T03:32:16,696 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 756be66b118f1073fd36299191edec1f: Waiting for close lock at 1731468736695Running coprocessor pre-close hooks at 1731468736695Disabling compacts and flushes for region at 1731468736695Disabling writes for close at 1731468736695Writing region close event to WAL at 1731468736695Running coprocessor post-close hooks at 1731468736695Closed at 1731468736695 2024-11-13T03:32:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741908_1086 (size=3941) 2024-11-13T03:32:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741908_1086 (size=3941) 2024-11-13T03:32:16,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741908_1086 (size=3941) 2024-11-13T03:32:16,713 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, size=3.8 K (3941bytes) 2024-11-13T03:32:16,713 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 2024-11-13T03:32:16,714 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 after 1ms 2024-11-13T03:32:16,716 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 took 3ms 2024-11-13T03:32:16,718 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 so closing down 2024-11-13T03:32:16,718 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:16,720 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1731468727566.temp 2024-11-13T03:32:16,721 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp 2024-11-13T03:32:16,721 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741909_1087 (size=2983) 2024-11-13T03:32:16,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741909_1087 (size=2983) 2024-11-13T03:32:16,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741909_1087 (size=2983) 2024-11-13T03:32:16,727 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:16,728 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 2024-11-13T03:32:16,728 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, size=3.8 K, length=3941, corrupted=false, cancelled=false 2024-11-13T03:32:16,728 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, journal: Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, size=3.8 K (3941bytes) at 1731468736713Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 so closing down at 1731468736718 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp at 1731468736721 (+3 ms)3 split writer threads finished at 1731468736721Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1731468736727 (+6 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000004-wal.1731468727566.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 at 1731468736728 (+1 ms)Processed 23 edits across 1 Regions in 11 ms; skipped=3; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566, size=3.8 K, length=3941, corrupted=false, cancelled=false at 1731468736728 2024-11-13T03:32:16,730 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468727566 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468727566 2024-11-13T03:32:16,731 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 2024-11-13T03:32:16,731 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:16,732 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:16,744 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468736732, exclude list is [], retry=0 2024-11-13T03:32:16,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:16,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:16,747 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:16,749 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468736732 2024-11-13T03:32:16,750 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:32:16,751 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 756be66b118f1073fd36299191edec1f, NAME => 'testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:16,752 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,752 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:16,752 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,752 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,754 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,755 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName a 2024-11-13T03:32:16,755 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:16,756 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:16,756 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,758 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName b 2024-11-13T03:32:16,758 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:16,759 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:16,759 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,760 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 756be66b118f1073fd36299191edec1f columnFamilyName c 2024-11-13T03:32:16,760 DEBUG [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:16,761 INFO [StoreOpener-756be66b118f1073fd36299191edec1f-1 {}] regionserver.HStore(327): Store=756be66b118f1073fd36299191edec1f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:16,761 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,762 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,763 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,763 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 2024-11-13T03:32:16,767 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 2024-11-13T03:32:16,767 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 756be66b118f1073fd36299191edec1f 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-11-13T03:32:16,782 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/a/574d1f7f11c5478f9114ed1ea12187b0 is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1731468736688/Put/seqid=0 2024-11-13T03:32:16,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741911_1089 (size=5523) 2024-11-13T03:32:16,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741911_1089 (size=5523) 2024-11-13T03:32:16,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741911_1089 (size=5523) 2024-11-13T03:32:16,832 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/a/574d1f7f11c5478f9114ed1ea12187b0 2024-11-13T03:32:16,852 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/b/710db750ac5d4b64af00054d4247e03a is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1731468736685/Put/seqid=0 2024-11-13T03:32:16,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741912_1090 (size=5524) 2024-11-13T03:32:16,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741912_1090 (size=5524) 2024-11-13T03:32:16,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741912_1090 (size=5524) 2024-11-13T03:32:16,860 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/b/710db750ac5d4b64af00054d4247e03a 2024-11-13T03:32:16,884 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/c/246ae5b1032a4f77affc05b43060f4b5 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1731468736686/Put/seqid=0 2024-11-13T03:32:16,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741913_1091 (size=5457) 2024-11-13T03:32:16,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741913_1091 (size=5457) 2024-11-13T03:32:16,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741913_1091 (size=5457) 2024-11-13T03:32:16,892 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/c/246ae5b1032a4f77affc05b43060f4b5 2024-11-13T03:32:16,898 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/a/574d1f7f11c5478f9114ed1ea12187b0 as hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/a/574d1f7f11c5478f9114ed1ea12187b0 2024-11-13T03:32:16,904 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/a/574d1f7f11c5478f9114ed1ea12187b0, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T03:32:16,905 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/b/710db750ac5d4b64af00054d4247e03a as hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/b/710db750ac5d4b64af00054d4247e03a 2024-11-13T03:32:16,911 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/b/710db750ac5d4b64af00054d4247e03a, entries=7, sequenceid=26, filesize=5.4 K 2024-11-13T03:32:16,912 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/.tmp/c/246ae5b1032a4f77affc05b43060f4b5 as hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/c/246ae5b1032a4f77affc05b43060f4b5 2024-11-13T03:32:16,917 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/c/246ae5b1032a4f77affc05b43060f4b5, entries=6, sequenceid=26, filesize=5.3 K 2024-11-13T03:32:16,918 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 756be66b118f1073fd36299191edec1f in 151ms, sequenceid=26, compaction requested=false; wal=null 2024-11-13T03:32:16,919 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/0000000000000000026 2024-11-13T03:32:16,920 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,920 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,921 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:16,922 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,924 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsAfterAbortingFlush/756be66b118f1073fd36299191edec1f/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-11-13T03:32:16,925 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 756be66b118f1073fd36299191edec1f; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73197801, jitterRate=0.09073223173618317}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:16,925 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 756be66b118f1073fd36299191edec1f 2024-11-13T03:32:16,926 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 756be66b118f1073fd36299191edec1f: Running coprocessor pre-open hook at 1731468736752Writing region info on filesystem at 1731468736752Initializing all the Stores at 1731468736753 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468736753Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468736754 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468736754Obtaining lock to block concurrent updates at 1731468736767 (+13 ms)Preparing flush snapshotting stores in 756be66b118f1073fd36299191edec1f at 1731468736767Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1731468736767Flushing stores of testReplayEditsAfterAbortingFlush,,1731468727495.756be66b118f1073fd36299191edec1f. at 1731468736767Flushing 756be66b118f1073fd36299191edec1f/a: creating writer at 1731468736767Flushing 756be66b118f1073fd36299191edec1f/a: appending metadata at 1731468736782 (+15 ms)Flushing 756be66b118f1073fd36299191edec1f/a: closing flushed file at 1731468736782Flushing 756be66b118f1073fd36299191edec1f/b: creating writer at 1731468736837 (+55 ms)Flushing 756be66b118f1073fd36299191edec1f/b: appending metadata at 1731468736851 (+14 ms)Flushing 756be66b118f1073fd36299191edec1f/b: closing flushed file at 1731468736851Flushing 756be66b118f1073fd36299191edec1f/c: creating writer at 1731468736866 (+15 ms)Flushing 756be66b118f1073fd36299191edec1f/c: appending metadata at 1731468736884 (+18 ms)Flushing 756be66b118f1073fd36299191edec1f/c: closing flushed file at 1731468736884Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@50f83bc9: reopening flushed file at 1731468736897 (+13 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c147b: reopening flushed file at 1731468736904 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@213cd7a3: reopening flushed file at 1731468736911 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for 756be66b118f1073fd36299191edec1f in 151ms, sequenceid=26, compaction requested=false; wal=null at 1731468736918 (+7 ms)Cleaning up temporary data from old regions at 1731468736920 (+2 ms)Running coprocessor post-open hooks at 1731468736925 (+5 ms)Region opened successfully at 1731468736926 (+1 ms) 2024-11-13T03:32:16,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=419 (was 418) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:43960 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33736 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:43946 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38904 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33750 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741910_1088] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38918 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1120) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 299), ProcessCount=11 (was 11), AvailableMemoryMB=2861 (was 2882) 2024-11-13T03:32:16,945 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T03:32:16,959 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=419, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=2860 2024-11-13T03:32:16,959 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-11-13T03:32:16,980 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:16,982 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:16,983 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:16,987 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-63101668, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-63101668, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:17,003 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-63101668/hregion-63101668.1731468736987, exclude list is [], retry=0 2024-11-13T03:32:17,006 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:17,006 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:17,006 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:17,009 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-63101668/hregion-63101668.1731468736987 2024-11-13T03:32:17,009 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:17,009 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 739e8c395ea4b1116ae1c4b551bb6bbb, NAME => 'testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:32:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741915_1093 (size=61) 2024-11-13T03:32:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741915_1093 (size=61) 2024-11-13T03:32:17,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741915_1093 (size=61) 2024-11-13T03:32:17,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,022 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,024 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739e8c395ea4b1116ae1c4b551bb6bbb columnFamilyName a 2024-11-13T03:32:17,024 DEBUG [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,024 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(327): Store=739e8c395ea4b1116ae1c4b551bb6bbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,024 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,025 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,026 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,028 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,030 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:17,031 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739e8c395ea4b1116ae1c4b551bb6bbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70268886, jitterRate=0.04708799719810486}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Writing region info on filesystem at 1731468737021Initializing all the Stores at 1731468737022 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737022Cleaning up temporary data from old regions at 1731468737026 (+4 ms)Region opened successfully at 1731468737031 (+5 ms) 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 739e8c395ea4b1116ae1c4b551bb6bbb, disabling compactions & flushes 2024-11-13T03:32:17,032 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. after waiting 0 ms 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,032 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,032 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Waiting for close lock at 1731468737032Disabling compacts and flushes for region at 1731468737032Disabling writes for close at 1731468737032Writing region close event to WAL at 1731468737032Closed at 1731468737032 2024-11-13T03:32:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741914_1092 (size=93) 2024-11-13T03:32:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741914_1092 (size=93) 2024-11-13T03:32:17,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741914_1092 (size=93) 2024-11-13T03:32:17,040 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:17,040 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-63101668:(num 1731468736987) 2024-11-13T03:32:17,040 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:17,042 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:17,061 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, exclude list is [], retry=0 2024-11-13T03:32:17,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:17,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:17,064 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:17,066 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 2024-11-13T03:32:17,066 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:17,066 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 739e8c395ea4b1116ae1c4b551bb6bbb, NAME => 'testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:17,067 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,067 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,067 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,068 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,069 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739e8c395ea4b1116ae1c4b551bb6bbb columnFamilyName a 2024-11-13T03:32:17,069 DEBUG [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,069 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(327): Store=739e8c395ea4b1116ae1c4b551bb6bbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,070 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,070 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,071 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,072 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,072 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,073 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,074 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739e8c395ea4b1116ae1c4b551bb6bbb; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59457421, jitterRate=-0.11401538550853729}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:17,075 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Writing region info on filesystem at 1731468737067Initializing all the Stores at 1731468737068 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737068Cleaning up temporary data from old regions at 1731468737072 (+4 ms)Region opened successfully at 1731468737075 (+3 ms) 2024-11-13T03:32:17,082 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 739e8c395ea4b1116ae1c4b551bb6bbb, disabling compactions & flushes 2024-11-13T03:32:17,082 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,082 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,082 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. after waiting 0 ms 2024-11-13T03:32:17,082 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,082 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,082 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. 2024-11-13T03:32:17,082 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Waiting for close lock at 1731468737082Disabling compacts and flushes for region at 1731468737082Disabling writes for close at 1731468737082Writing region close event to WAL at 1731468737082Closed at 1731468737082 2024-11-13T03:32:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741916_1094 (size=1623) 2024-11-13T03:32:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741916_1094 (size=1623) 2024-11-13T03:32:17,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741916_1094 (size=1623) 2024-11-13T03:32:17,100 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, size=1.6 K (1623bytes) 2024-11-13T03:32:17,100 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 2024-11-13T03:32:17,101 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 after 1ms 2024-11-13T03:32:17,103 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 took 3ms 2024-11-13T03:32:17,105 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 so closing down 2024-11-13T03:32:17,105 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:17,106 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468737043.temp 2024-11-13T03:32:17,107 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp 2024-11-13T03:32:17,107 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:17,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741917_1095 (size=1623) 2024-11-13T03:32:17,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741917_1095 (size=1623) 2024-11-13T03:32:17,116 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741917_1095 (size=1623) 2024-11-13T03:32:17,116 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:17,117 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp to hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 2024-11-13T03:32:17,118 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, size=1.6 K, length=1623, corrupted=false, cancelled=false 2024-11-13T03:32:17,118 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, journal: Splitting hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, size=1.6 K (1623bytes) at 1731468737100Finishing writing output for hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 so closing down at 1731468737105 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp at 1731468737107 (+2 ms)3 split writer threads finished at 1731468737107Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1731468737116 (+9 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000003-wal.1731468737043.temp to hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 at 1731468737117 (+1 ms)Processed 10 edits across 1 Regions in 14 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043, size=1.6 K, length=1623, corrupted=false, cancelled=false at 1731468737118 (+1 ms) 2024-11-13T03:32:17,119 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737043 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468737043 2024-11-13T03:32:17,120 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 2024-11-13T03:32:17,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-11-13T03:32:17,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:17,499 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:17,502 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:17,515 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737502, exclude list is [], retry=0 2024-11-13T03:32:17,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:17,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:17,518 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:17,520 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737502 2024-11-13T03:32:17,520 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:32:17,521 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 739e8c395ea4b1116ae1c4b551bb6bbb, NAME => 'testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:17,521 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,521 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,521 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,524 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,525 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739e8c395ea4b1116ae1c4b551bb6bbb columnFamilyName a 2024-11-13T03:32:17,526 DEBUG [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,526 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(327): Store=739e8c395ea4b1116ae1c4b551bb6bbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,527 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,527 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,529 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,529 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 2024-11-13T03:32:17,532 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 2024-11-13T03:32:17,532 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 739e8c395ea4b1116ae1c4b551bb6bbb 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-11-13T03:32:17,552 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/.tmp/a/8846ab4bf8564177a701bb73246439e4 is 79, key is testDatalossWhenInputError/a:x0/1731468737075/Put/seqid=0 2024-11-13T03:32:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741919_1097 (size=5808) 2024-11-13T03:32:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741919_1097 (size=5808) 2024-11-13T03:32:17,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741919_1097 (size=5808) 2024-11-13T03:32:17,559 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/.tmp/a/8846ab4bf8564177a701bb73246439e4 2024-11-13T03:32:17,569 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/.tmp/a/8846ab4bf8564177a701bb73246439e4 as hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/a/8846ab4bf8564177a701bb73246439e4 2024-11-13T03:32:17,579 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/a/8846ab4bf8564177a701bb73246439e4, entries=10, sequenceid=12, filesize=5.7 K 2024-11-13T03:32:17,579 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 739e8c395ea4b1116ae1c4b551bb6bbb in 47ms, sequenceid=12, compaction requested=false; wal=null 2024-11-13T03:32:17,580 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/0000000000000000012 2024-11-13T03:32:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,581 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,584 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,586 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-13T03:32:17,587 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739e8c395ea4b1116ae1c4b551bb6bbb; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66493301, jitterRate=-0.009172603487968445}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:17,588 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Writing region info on filesystem at 1731468737521Initializing all the Stores at 1731468737524 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737524Obtaining lock to block concurrent updates at 1731468737532 (+8 ms)Preparing flush snapshotting stores in 739e8c395ea4b1116ae1c4b551bb6bbb at 1731468737532Finished memstore snapshotting testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1731468737532Flushing stores of testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb. at 1731468737532Flushing 739e8c395ea4b1116ae1c4b551bb6bbb/a: creating writer at 1731468737532Flushing 739e8c395ea4b1116ae1c4b551bb6bbb/a: appending metadata at 1731468737552 (+20 ms)Flushing 739e8c395ea4b1116ae1c4b551bb6bbb/a: closing flushed file at 1731468737552Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7d04dbcc: reopening flushed file at 1731468737567 (+15 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 739e8c395ea4b1116ae1c4b551bb6bbb in 47ms, sequenceid=12, compaction requested=false; wal=null at 1731468737579 (+12 ms)Cleaning up temporary data from old regions at 1731468737581 (+2 ms)Region opened successfully at 1731468737588 (+7 ms) 2024-11-13T03:32:17,592 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 739e8c395ea4b1116ae1c4b551bb6bbb, NAME => 'testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:17,592 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1731468736980.739e8c395ea4b1116ae1c4b551bb6bbb.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,592 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,592 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,593 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,594 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 739e8c395ea4b1116ae1c4b551bb6bbb columnFamilyName a 2024-11-13T03:32:17,594 DEBUG [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,599 DEBUG [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/a/8846ab4bf8564177a701bb73246439e4 2024-11-13T03:32:17,599 INFO [StoreOpener-739e8c395ea4b1116ae1c4b551bb6bbb-1 {}] regionserver.HStore(327): Store=739e8c395ea4b1116ae1c4b551bb6bbb/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,599 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,599 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,600 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,601 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,601 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,603 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 739e8c395ea4b1116ae1c4b551bb6bbb 2024-11-13T03:32:17,605 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testDatalossWhenInputError/739e8c395ea4b1116ae1c4b551bb6bbb/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-11-13T03:32:17,606 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 739e8c395ea4b1116ae1c4b551bb6bbb; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65689704, jitterRate=-0.02114713191986084}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-13T03:32:17,606 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 739e8c395ea4b1116ae1c4b551bb6bbb: Writing region info on filesystem at 1731468737592Initializing all the Stores at 1731468737593 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737593Cleaning up temporary data from old regions at 1731468737601 (+8 ms)Region opened successfully at 1731468737606 (+5 ms) 2024-11-13T03:32:17,625 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testDatalossWhenInputError Thread=429 (was 419) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:43960 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33736 [Waiting for operation #8] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:44020 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33822 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:38918 [Waiting for operation #21] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:39004 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741918_1096] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=268 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=2805 (was 2860) 2024-11-13T03:32:17,625 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T03:32:17,637 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=429, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=268, ProcessCount=11, AvailableMemoryMB=2804 2024-11-13T03:32:17,637 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-11-13T03:32:17,651 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:17,653 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:17,654 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:17,656 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-59824174, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-59824174, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:17,690 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-59824174/hregion-59824174.1731468737675, exclude list is [], retry=0 2024-11-13T03:32:17,693 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:17,693 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:17,693 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:17,696 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-59824174/hregion-59824174.1731468737675 2024-11-13T03:32:17,697 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:17,697 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 98e1619b8d72ce83928ca15ff8ea6ee5, NAME => 'testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:32:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741921_1099 (size=63) 2024-11-13T03:32:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741921_1099 (size=63) 2024-11-13T03:32:17,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741921_1099 (size=63) 2024-11-13T03:32:17,707 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,708 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,709 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName a 2024-11-13T03:32:17,709 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,709 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,710 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,711 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName b 2024-11-13T03:32:17,711 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,711 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,711 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,712 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName c 2024-11-13T03:32:17,713 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,713 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,713 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,714 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,715 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,715 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,716 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:17,717 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,719 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:17,719 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 98e1619b8d72ce83928ca15ff8ea6ee5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74532060, jitterRate=0.11061424016952515}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 98e1619b8d72ce83928ca15ff8ea6ee5: Writing region info on filesystem at 1731468737707Initializing all the Stores at 1731468737707Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737707Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737708 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737708Cleaning up temporary data from old regions at 1731468737715 (+7 ms)Region opened successfully at 1731468737720 (+5 ms) 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 98e1619b8d72ce83928ca15ff8ea6ee5, disabling compactions & flushes 2024-11-13T03:32:17,720 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. after waiting 0 ms 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:17,720 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:17,720 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 98e1619b8d72ce83928ca15ff8ea6ee5: Waiting for close lock at 1731468737720Disabling compacts and flushes for region at 1731468737720Disabling writes for close at 1731468737720Writing region close event to WAL at 1731468737720Closed at 1731468737720 2024-11-13T03:32:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741920_1098 (size=93) 2024-11-13T03:32:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741920_1098 (size=93) 2024-11-13T03:32:17,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741920_1098 (size=93) 2024-11-13T03:32:17,725 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:17,725 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-59824174:(num 1731468737675) 2024-11-13T03:32:17,725 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:17,727 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:17,739 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, exclude list is [], retry=0 2024-11-13T03:32:17,742 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:17,743 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:17,743 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:17,744 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 2024-11-13T03:32:17,744 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 98e1619b8d72ce83928ca15ff8ea6ee5, NAME => 'testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,745 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,749 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,750 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName a 2024-11-13T03:32:17,750 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,750 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,750 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,751 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName b 2024-11-13T03:32:17,751 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,752 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,752 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,752 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName c 2024-11-13T03:32:17,752 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:17,753 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:17,753 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,754 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,755 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,756 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,756 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,756 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:17,757 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:17,758 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 98e1619b8d72ce83928ca15ff8ea6ee5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58771573, jitterRate=-0.12423531711101532}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:17,759 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 98e1619b8d72ce83928ca15ff8ea6ee5: Writing region info on filesystem at 1731468737745Initializing all the Stores at 1731468737746 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737746Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737748 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468737748Cleaning up temporary data from old regions at 1731468737756 (+8 ms)Region opened successfully at 1731468737759 (+3 ms) 2024-11-13T03:32:17,763 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1731468737763/Put/seqid=0 2024-11-13T03:32:17,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741923_1101 (size=4875) 2024-11-13T03:32:17,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741923_1101 (size=4875) 2024-11-13T03:32:17,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741923_1101 (size=4875) 2024-11-13T03:32:17,774 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1731468737774/Put/seqid=0 2024-11-13T03:32:17,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741924_1102 (size=4875) 2024-11-13T03:32:17,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741924_1102 (size=4875) 2024-11-13T03:32:17,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741924_1102 (size=4875) 2024-11-13T03:32:17,788 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1731468737788/Put/seqid=0 2024-11-13T03:32:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741925_1103 (size=4875) 2024-11-13T03:32:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741925_1103 (size=4875) 2024-11-13T03:32:17,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741925_1103 (size=4875) 2024-11-13T03:32:17,798 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,801 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-11-13T03:32:17,801 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:32:17,801 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,804 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-11-13T03:32:17,804 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:32:17,805 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,807 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-11-13T03:32:17,807 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-11-13T03:32:17,808 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 98e1619b8d72ce83928ca15ff8ea6ee5 3/3 column families, dataSize=51 B heapSize=896 B 2024-11-13T03:32:17,823 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp/a/dc479275760e44478b40655f7fe1474e is 55, key is testCompactedBulkLoadedFiles/a:a/1731468737759/Put/seqid=0 2024-11-13T03:32:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741926_1104 (size=5107) 2024-11-13T03:32:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741926_1104 (size=5107) 2024-11-13T03:32:17,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741926_1104 (size=5107) 2024-11-13T03:32:17,830 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp/a/dc479275760e44478b40655f7fe1474e 2024-11-13T03:32:17,835 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp/a/dc479275760e44478b40655f7fe1474e as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e 2024-11-13T03:32:17,840 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e, entries=1, sequenceid=4, filesize=5.0 K 2024-11-13T03:32:17,841 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for 98e1619b8d72ce83928ca15ff8ea6ee5 in 32ms, sequenceid=4, compaction requested=false 2024-11-13T03:32:17,841 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 98e1619b8d72ce83928ca15ff8ea6ee5: 2024-11-13T03:32:17,842 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ 2024-11-13T03:32:17,843 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ 2024-11-13T03:32:17,844 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ 2024-11-13T03:32:17,845 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile0 into 98e1619b8d72ce83928ca15ff8ea6ee5/a as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ - updating store file list. 2024-11-13T03:32:17,850 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for a5bc115522944b00ac31369b6e3c4f88_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:17,850 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ into 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,850 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile0 into 98e1619b8d72ce83928ca15ff8ea6ee5/a (new location: hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_) 2024-11-13T03:32:17,851 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile1 into 98e1619b8d72ce83928ca15ff8ea6ee5/a as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ - updating store file list. 2024-11-13T03:32:17,856 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0e2e34454467444a8e136283b0d44d14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:17,856 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ into 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,856 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile1 into 98e1619b8d72ce83928ca15ff8ea6ee5/a (new location: hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_) 2024-11-13T03:32:17,857 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile2 into 98e1619b8d72ce83928ca15ff8ea6ee5/a as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ - updating store file list. 2024-11-13T03:32:17,863 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for e51d33b4c62644d393b45531297c6494_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:17,863 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ into 98e1619b8d72ce83928ca15ff8ea6ee5/a 2024-11-13T03:32:17,863 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:33705/hbase/testCompactedBulkLoadedFiles/hfile2 into 98e1619b8d72ce83928ca15ff8ea6ee5/a (new location: hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_) 2024-11-13T03:32:17,872 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-13T03:32:17,872 DEBUG [Time-limited test {}] regionserver.HStore(1541): 98e1619b8d72ce83928ca15ff8ea6ee5/a is initiating major compaction (all files) 2024-11-13T03:32:17,872 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 98e1619b8d72ce83928ca15ff8ea6ee5/a in testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:17,872 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_] into tmpdir=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp, totalSize=19.3 K 2024-11-13T03:32:17,873 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting dc479275760e44478b40655f7fe1474e, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1731468737759 2024-11-13T03:32:17,874 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting a5bc115522944b00ac31369b6e3c4f88_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:32:17,874 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 0e2e34454467444a8e136283b0d44d14_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:32:17,874 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting e51d33b4c62644d393b45531297c6494_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-11-13T03:32:17,888 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp/a/0ba535d2bf4844bc9fa3579f064e4c69 is 55, key is testCompactedBulkLoadedFiles/a:a/1731468737759/Put/seqid=0 2024-11-13T03:32:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741927_1105 (size=6154) 2024-11-13T03:32:17,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741927_1105 (size=6154) 2024-11-13T03:32:17,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741927_1105 (size=6154) 2024-11-13T03:32:17,899 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/.tmp/a/0ba535d2bf4844bc9fa3579f064e4c69 as hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0ba535d2bf4844bc9fa3579f064e4c69 2024-11-13T03:32:17,905 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in 98e1619b8d72ce83928ca15ff8ea6ee5/a of 98e1619b8d72ce83928ca15ff8ea6ee5 into 0ba535d2bf4844bc9fa3579f064e4c69(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-13T03:32:17,906 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 98e1619b8d72ce83928ca15ff8ea6ee5: 2024-11-13T03:32:17,906 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T03:32:17,906 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-11-13T03:32:17,936 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, size=0 (0bytes) 2024-11-13T03:32:17,936 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 might be still open, length is 0 2024-11-13T03:32:17,936 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 2024-11-13T03:32:17,937 WARN [IPC Server handler 1 on default port 33705 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 has not been closed. Lease recovery is in progress. RecoveryId = 1106 for block blk_1073741922_1100 2024-11-13T03:32:17,937 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 after 1ms 2024-11-13T03:32:18,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33848 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33848 dst: /127.0.0.1:43451 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43451 remote=/127.0.0.1:33848]. Total timeout mills is 60000, 58962 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:18,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:39026 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:37801:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:39026 dst: /127.0.0.1:37801 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:18,944 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:44058 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741922_1100] {}] datanode.DataXceiver(331): 127.0.0.1:43285:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:44058 dst: /127.0.0.1:43285 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:18,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741922_1106 (size=1659) 2024-11-13T03:32:18,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741922_1106 (size=1659) 2024-11-13T03:32:21,938 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 after 4002ms 2024-11-13T03:32:21,942 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 took 4006ms 2024-11-13T03:32:21,945 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727; continuing. 2024-11-13T03:32:21,945 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 so closing down 2024-11-13T03:32:21,946 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:21,948 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468737727.temp 2024-11-13T03:32:21,949 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp 2024-11-13T03:32:21,950 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:21,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741928_1107 (size=698) 2024-11-13T03:32:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741928_1107 (size=698) 2024-11-13T03:32:21,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741928_1107 (size=698) 2024-11-13T03:32:21,958 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:21,960 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp to hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 2024-11-13T03:32:21,960 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:32:21,960 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, journal: Splitting hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, size=0 (0bytes) at 1731468737936Finishing writing output for hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 so closing down at 1731468741946 (+4010 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp at 1731468741949 (+3 ms)3 split writer threads finished at 1731468741950 (+1 ms)Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1731468741958 (+8 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000003-wal.1731468737727.temp to hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 at 1731468741960 (+2 ms)Processed 5 edits across 1 Regions in 17 ms; skipped=3; WAL=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727, size=0, length=0, corrupted=false, cancelled=false at 1731468741960 2024-11-13T03:32:21,962 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468737727 2024-11-13T03:32:21,963 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 2024-11-13T03:32:21,963 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:21,965 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:21,977 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468741965, exclude list is [], retry=0 2024-11-13T03:32:21,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:21,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:21,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:21,981 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468741965 2024-11-13T03:32:21,982 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:21,982 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 98e1619b8d72ce83928ca15ff8ea6ee5, NAME => 'testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:21,982 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:21,982 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:21,982 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:21,983 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:21,984 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName a 2024-11-13T03:32:21,985 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:21,991 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0ba535d2bf4844bc9fa3579f064e4c69 2024-11-13T03:32:21,994 DEBUG [StoreFileOpener-98e1619b8d72ce83928ca15ff8ea6ee5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 0e2e34454467444a8e136283b0d44d14_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:21,995 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ 2024-11-13T03:32:21,999 DEBUG [StoreFileOpener-98e1619b8d72ce83928ca15ff8ea6ee5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for a5bc115522944b00ac31369b6e3c4f88_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:21,999 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ 2024-11-13T03:32:22,003 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e 2024-11-13T03:32:22,008 DEBUG [StoreFileOpener-98e1619b8d72ce83928ca15ff8ea6ee5-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for e51d33b4c62644d393b45531297c6494_SeqId_4_: NONE, but ROW specified in column family configuration 2024-11-13T03:32:22,008 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ 2024-11-13T03:32:22,008 WARN [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@38a8cabe 2024-11-13T03:32:22,008 WARN [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@38a8cabe 2024-11-13T03:32:22,008 WARN [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@38a8cabe 2024-11-13T03:32:22,008 WARN [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@38a8cabe 2024-11-13T03:32:22,008 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_] to archive 2024-11-13T03:32:22,009 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-13T03:32:22,011 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ to hdfs://localhost:33705/hbase/archive/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/0e2e34454467444a8e136283b0d44d14_SeqId_4_ 2024-11-13T03:32:22,012 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ to hdfs://localhost:33705/hbase/archive/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/a5bc115522944b00ac31369b6e3c4f88_SeqId_4_ 2024-11-13T03:32:22,013 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e to hdfs://localhost:33705/hbase/archive/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/dc479275760e44478b40655f7fe1474e 2024-11-13T03:32:22,014 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ to hdfs://localhost:33705/hbase/archive/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/a/e51d33b4c62644d393b45531297c6494_SeqId_4_ 2024-11-13T03:32:22,014 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,015 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,015 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName b 2024-11-13T03:32:22,015 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,016 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,016 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,016 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 98e1619b8d72ce83928ca15ff8ea6ee5 columnFamilyName c 2024-11-13T03:32:22,016 DEBUG [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,017 INFO [StoreOpener-98e1619b8d72ce83928ca15ff8ea6ee5-1 {}] regionserver.HStore(327): Store=98e1619b8d72ce83928ca15ff8ea6ee5/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,017 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,017 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,019 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,019 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 2024-11-13T03:32:22,022 DEBUG [Time-limited test {}] regionserver.HRegion(5836): 98e1619b8d72ce83928ca15ff8ea6ee5 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "98e1619b8d72ce83928ca15ff8ea6ee5" family_name: "a" compaction_input: "dc479275760e44478b40655f7fe1474e" compaction_input: "a5bc115522944b00ac31369b6e3c4f88_SeqId_4_" compaction_input: "0e2e34454467444a8e136283b0d44d14_SeqId_4_" compaction_input: "e51d33b4c62644d393b45531297c6494_SeqId_4_" compaction_output: "0ba535d2bf4844bc9fa3579f064e4c69" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-11-13T03:32:22,022 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-11-13T03:32:22,022 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 2024-11-13T03:32:22,023 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/0000000000000000008 2024-11-13T03:32:22,024 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,024 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,024 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:22,025 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 98e1619b8d72ce83928ca15ff8ea6ee5 2024-11-13T03:32:22,027 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testCompactedBulkLoadedFiles/98e1619b8d72ce83928ca15ff8ea6ee5/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-11-13T03:32:22,028 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 98e1619b8d72ce83928ca15ff8ea6ee5; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74668785, jitterRate=0.11265160143375397}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:22,029 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 98e1619b8d72ce83928ca15ff8ea6ee5: Writing region info on filesystem at 1731468741982Initializing all the Stores at 1731468741983 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468741983Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468741983Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468741983Cleaning up temporary data from old regions at 1731468742024 (+41 ms)Region opened successfully at 1731468742028 (+4 ms) 2024-11-13T03:32:22,031 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 98e1619b8d72ce83928ca15ff8ea6ee5, disabling compactions & flushes 2024-11-13T03:32:22,031 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:22,031 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:22,031 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. after waiting 0 ms 2024-11-13T03:32:22,031 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:22,032 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1731468737652.98e1619b8d72ce83928ca15ff8ea6ee5. 2024-11-13T03:32:22,032 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 98e1619b8d72ce83928ca15ff8ea6ee5: Waiting for close lock at 1731468742031Disabling compacts and flushes for region at 1731468742031Disabling writes for close at 1731468742031Writing region close event to WAL at 1731468742032 (+1 ms)Closed at 1731468742032 2024-11-13T03:32:22,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741929_1108 (size=93) 2024-11-13T03:32:22,034 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468741965 not finished, retry = 0 2024-11-13T03:32:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741929_1108 (size=93) 2024-11-13T03:32:22,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741929_1108 (size=93) 2024-11-13T03:32:22,137 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:22,137 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468741965) 2024-11-13T03:32:22,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testCompactedBulkLoadedFiles Thread=443 (was 429) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-337918613_22 at /127.0.0.1:33604 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:33705 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33705 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-337918613_22 at /127.0.0.1:58960 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-337918613_22 at /127.0.0.1:35672 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1348 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=246 (was 268), ProcessCount=11 (was 11), AvailableMemoryMB=2738 (was 2804) 2024-11-13T03:32:22,152 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T03:32:22,164 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=443, OpenFileDescriptor=1348, MaxFileDescriptor=1048576, SystemLoadAverage=246, ProcessCount=11, AvailableMemoryMB=2737 2024-11-13T03:32:22,164 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1348 is superior to 1024 2024-11-13T03:32:22,178 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:22,180 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-13T03:32:22,181 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-13T03:32:22,183 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-88103888, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/hregion-88103888, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:22,200 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-13T03:32:22,202 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-88103888/hregion-88103888.1731468742184, exclude list is [], retry=0 2024-11-13T03:32:22,204 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:22,204 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:22,205 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:22,207 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-88103888/hregion-88103888.1731468742184 2024-11-13T03:32:22,207 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:40391:40391)] 2024-11-13T03:32:22,208 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 62493399fe54f1723fc2dc03d994d24a, NAME => 'testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:33705/hbase 2024-11-13T03:32:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741931_1110 (size=67) 2024-11-13T03:32:22,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741931_1110 (size=67) 2024-11-13T03:32:22,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741931_1110 (size=67) 2024-11-13T03:32:22,223 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:22,228 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,229 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName a 2024-11-13T03:32:22,229 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,230 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,230 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,231 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName b 2024-11-13T03:32:22,231 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,232 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,232 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,233 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName c 2024-11-13T03:32:22,233 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,233 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,234 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,234 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,235 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,235 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,236 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:22,236 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,238 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-13T03:32:22,239 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62493399fe54f1723fc2dc03d994d24a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72559661, jitterRate=0.08122320473194122}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62493399fe54f1723fc2dc03d994d24a: Writing region info on filesystem at 1731468742223Initializing all the Stores at 1731468742224 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742224Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742228 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742228Cleaning up temporary data from old regions at 1731468742235 (+7 ms)Region opened successfully at 1731468742239 (+4 ms) 2024-11-13T03:32:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62493399fe54f1723fc2dc03d994d24a, disabling compactions & flushes 2024-11-13T03:32:22,239 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. after waiting 0 ms 2024-11-13T03:32:22,239 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,239 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,240 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62493399fe54f1723fc2dc03d994d24a: Waiting for close lock at 1731468742239Disabling compacts and flushes for region at 1731468742239Disabling writes for close at 1731468742239Writing region close event to WAL at 1731468742239Closed at 1731468742239 2024-11-13T03:32:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741930_1109 (size=93) 2024-11-13T03:32:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741930_1109 (size=93) 2024-11-13T03:32:22,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741930_1109 (size=93) 2024-11-13T03:32:22,244 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:22,244 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-88103888:(num 1731468742184) 2024-11-13T03:32:22,244 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:22,246 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:22,258 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, exclude list is [], retry=0 2024-11-13T03:32:22,261 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:22,261 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:22,262 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:22,263 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 2024-11-13T03:32:22,263 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:22,263 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62493399fe54f1723fc2dc03d994d24a, NAME => 'testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:22,263 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:22,263 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,264 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,265 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,266 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName a 2024-11-13T03:32:22,266 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,266 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,266 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,267 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName b 2024-11-13T03:32:22,267 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,268 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,268 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,268 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName c 2024-11-13T03:32:22,268 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,269 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,269 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,270 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,271 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,272 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,272 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,272 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:22,274 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,275 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62493399fe54f1723fc2dc03d994d24a; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69711558, jitterRate=0.03878316283226013}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:22,275 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62493399fe54f1723fc2dc03d994d24a: Writing region info on filesystem at 1731468742264Initializing all the Stores at 1731468742264Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742265 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742265Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742265Cleaning up temporary data from old regions at 1731468742272 (+7 ms)Region opened successfully at 1731468742275 (+3 ms) 2024-11-13T03:32:22,283 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62493399fe54f1723fc2dc03d994d24a 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-11-13T03:32:22,305 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/f6e21c796a364889af5a154136520b90 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1731468742275/Put/seqid=0 2024-11-13T03:32:22,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741933_1112 (size=5958) 2024-11-13T03:32:22,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741933_1112 (size=5958) 2024-11-13T03:32:22,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741933_1112 (size=5958) 2024-11-13T03:32:22,314 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/f6e21c796a364889af5a154136520b90 2024-11-13T03:32:22,321 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/f6e21c796a364889af5a154136520b90 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/f6e21c796a364889af5a154136520b90 2024-11-13T03:32:22,326 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/f6e21c796a364889af5a154136520b90, entries=10, sequenceid=13, filesize=5.8 K 2024-11-13T03:32:22,327 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 62493399fe54f1723fc2dc03d994d24a in 44ms, sequenceid=13, compaction requested=false 2024-11-13T03:32:22,327 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 62493399fe54f1723fc2dc03d994d24a: 2024-11-13T03:32:22,344 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62493399fe54f1723fc2dc03d994d24a, disabling compactions & flushes 2024-11-13T03:32:22,344 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,344 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,344 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. after waiting 0 ms 2024-11-13T03:32:22,345 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,345 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,345 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:22,346 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62493399fe54f1723fc2dc03d994d24a: Waiting for close lock at 1731468742344Disabling compacts and flushes for region at 1731468742344Disabling writes for close at 1731468742345 (+1 ms)Writing region close event to WAL at 1731468742345Closed at 1731468742345 2024-11-13T03:32:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741932_1111 (size=5869) 2024-11-13T03:32:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741932_1111 (size=5869) 2024-11-13T03:32:22,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741932_1111 (size=5869) 2024-11-13T03:32:22,363 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, size=5.7 K (5869bytes) 2024-11-13T03:32:22,363 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 2024-11-13T03:32:22,364 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 after 1ms 2024-11-13T03:32:22,366 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 took 3ms 2024-11-13T03:32:22,368 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 so closing down 2024-11-13T03:32:22,368 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:22,369 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1731468742246.temp 2024-11-13T03:32:22,370 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp 2024-11-13T03:32:22,370 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:22,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741934_1113 (size=5223) 2024-11-13T03:32:22,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741934_1113 (size=5223) 2024-11-13T03:32:22,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741934_1113 (size=5223) 2024-11-13T03:32:22,377 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:22,378 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 2024-11-13T03:32:22,378 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, size=5.7 K, length=5869, corrupted=false, cancelled=false 2024-11-13T03:32:22,378 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, journal: Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, size=5.7 K (5869bytes) at 1731468742363Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 so closing down at 1731468742368 (+5 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp at 1731468742370 (+2 ms)3 split writer threads finished at 1731468742370Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468742377 (+7 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000003-wal.1731468742246.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 at 1731468742378 (+1 ms)Processed 32 edits across 1 Regions in 12 ms; skipped=2; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246, size=5.7 K, length=5869, corrupted=false, cancelled=false at 1731468742378 2024-11-13T03:32:22,380 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742246 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468742246 2024-11-13T03:32:22,381 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 2024-11-13T03:32:22,381 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:22,383 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:22,394 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, exclude list is [], retry=0 2024-11-13T03:32:22,396 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:22,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:22,397 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:22,398 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 2024-11-13T03:32:22,399 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:34497:34497),(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907)] 2024-11-13T03:32:22,399 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 62493399fe54f1723fc2dc03d994d24a, NAME => 'testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.', STARTKEY => '', ENDKEY => ''} 2024-11-13T03:32:22,399 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:22,399 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,399 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,400 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,401 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName a 2024-11-13T03:32:22,401 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,406 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/f6e21c796a364889af5a154136520b90 2024-11-13T03:32:22,406 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,407 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,407 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName b 2024-11-13T03:32:22,408 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,408 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,408 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,409 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName c 2024-11-13T03:32:22,409 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:22,410 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:22,410 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,410 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,412 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,413 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 2024-11-13T03:32:22,417 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 2024-11-13T03:32:22,417 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62493399fe54f1723fc2dc03d994d24a 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-11-13T03:32:22,433 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/5b12e8c4c023438abfbe239f01f28db0 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1731468742327/Put/seqid=0 2024-11-13T03:32:22,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741936_1115 (size=5958) 2024-11-13T03:32:22,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741936_1115 (size=5958) 2024-11-13T03:32:22,439 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/5b12e8c4c023438abfbe239f01f28db0 2024-11-13T03:32:22,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741936_1115 (size=5958) 2024-11-13T03:32:22,458 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/22e59e87182540c8b000a3c710808e07 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1731468742333/Put/seqid=0 2024-11-13T03:32:22,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741937_1116 (size=5958) 2024-11-13T03:32:22,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741937_1116 (size=5958) 2024-11-13T03:32:22,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741937_1116 (size=5958) 2024-11-13T03:32:22,464 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/22e59e87182540c8b000a3c710808e07 2024-11-13T03:32:22,470 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/5b12e8c4c023438abfbe239f01f28db0 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/b/5b12e8c4c023438abfbe239f01f28db0 2024-11-13T03:32:22,475 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/b/5b12e8c4c023438abfbe239f01f28db0, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T03:32:22,476 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/22e59e87182540c8b000a3c710808e07 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/c/22e59e87182540c8b000a3c710808e07 2024-11-13T03:32:22,480 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/c/22e59e87182540c8b000a3c710808e07, entries=10, sequenceid=35, filesize=5.8 K 2024-11-13T03:32:22,481 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 62493399fe54f1723fc2dc03d994d24a in 63ms, sequenceid=35, compaction requested=false; wal=null 2024-11-13T03:32:22,481 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000035 2024-11-13T03:32:22,482 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,482 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,483 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:22,484 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:22,486 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-11-13T03:32:22,487 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62493399fe54f1723fc2dc03d994d24a; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70963276, jitterRate=0.05743521451950073}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:22,487 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62493399fe54f1723fc2dc03d994d24a: Writing region info on filesystem at 1731468742399Initializing all the Stores at 1731468742400 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742400Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742400Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468742400Obtaining lock to block concurrent updates at 1731468742417 (+17 ms)Preparing flush snapshotting stores in 62493399fe54f1723fc2dc03d994d24a at 1731468742417Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1731468742417Flushing stores of testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. at 1731468742417Flushing 62493399fe54f1723fc2dc03d994d24a/b: creating writer at 1731468742417Flushing 62493399fe54f1723fc2dc03d994d24a/b: appending metadata at 1731468742433 (+16 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/b: closing flushed file at 1731468742433Flushing 62493399fe54f1723fc2dc03d994d24a/c: creating writer at 1731468742444 (+11 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/c: appending metadata at 1731468742457 (+13 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/c: closing flushed file at 1731468742457Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3a96cc69: reopening flushed file at 1731468742469 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73141f5b: reopening flushed file at 1731468742475 (+6 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 62493399fe54f1723fc2dc03d994d24a in 63ms, sequenceid=35, compaction requested=false; wal=null at 1731468742481 (+6 ms)Cleaning up temporary data from old regions at 1731468742482 (+1 ms)Region opened successfully at 1731468742487 (+5 ms) 2024-11-13T03:32:22,548 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, size=0 (0bytes) 2024-11-13T03:32:22,548 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 might be still open, length is 0 2024-11-13T03:32:22,548 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 2024-11-13T03:32:22,548 WARN [IPC Server handler 0 on default port 33705 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 has not been closed. Lease recovery is in progress. RecoveryId = 1117 for block blk_1073741935_1114 2024-11-13T03:32:22,549 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 after 1ms 2024-11-13T03:32:24,943 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:59018 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43451:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:59018 dst: /127.0.0.1:43451 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:43451 remote=/127.0.0.1:59018]. Total timeout mills is 60000, 57569 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:24,943 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:33678 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:43285:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:33678 dst: /127.0.0.1:43285 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:24,943 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_1314386333_22 at /127.0.0.1:35720 [Receiving block BP-648661827-172.17.0.3-1731468702443:blk_1073741935_1114] {}] datanode.DataXceiver(331): 127.0.0.1:37801:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:35720 dst: /127.0.0.1:37801 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:24,945 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741935_1117 (size=5215) 2024-11-13T03:32:24,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741935_1117 (size=5215) 2024-11-13T03:32:24,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741935_1117 (size=5215) 2024-11-13T03:32:26,550 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 after 4002ms 2024-11-13T03:32:26,556 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 took 4008ms 2024-11-13T03:32:26,560 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383; continuing. 2024-11-13T03:32:26,560 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 so closing down 2024-11-13T03:32:26,560 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-11-13T03:32:26,561 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1731468742383.temp 2024-11-13T03:32:26,562 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp 2024-11-13T03:32:26,562 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-11-13T03:32:26,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741938_1118 (size=5223) 2024-11-13T03:32:26,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741938_1118 (size=5223) 2024-11-13T03:32:26,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741938_1118 (size=5223) 2024-11-13T03:32:26,571 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-11-13T03:32:26,572 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 2024-11-13T03:32:26,572 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, size=0, length=0, corrupted=false, cancelled=false 2024-11-13T03:32:26,572 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, journal: Splitting hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, size=0 (0bytes) at 1731468742548Finishing writing output for hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 so closing down at 1731468746560 (+4012 ms)Creating recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp at 1731468746562 (+2 ms)3 split writer threads finished at 1731468746562Closed recovered edits writer path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1731468746571 (+9 ms)Rename recovered edits hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000037-wal.1731468742383.temp to hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 at 1731468746572 (+1 ms)Processed 30 edits across 1 Regions in 15 ms; skipped=0; WAL=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383, size=0, length=0, corrupted=false, cancelled=false at 1731468746572 2024-11-13T03:32:26,573 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 to hdfs://localhost:33705/hbase/oldWALs/wal.1731468742383 2024-11-13T03:32:26,574 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 2024-11-13T03:32:26,574 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-11-13T03:32:26,576 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:33705/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178, archiveDir=hdfs://localhost:33705/hbase/oldWALs, maxLogs=32 2024-11-13T03:32:26,588 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468746576, exclude list is [], retry=0 2024-11-13T03:32:26,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37801,DS-75a97f47-f4da-4c5c-ae35-34f12a772a01,DISK] 2024-11-13T03:32:26,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43285,DS-8d5ce0c4-eb86-455c-87fe-e979f874a76e,DISK] 2024-11-13T03:32:26,591 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43451,DS-0355b42a-550a-41e6-b90d-fde8ff4f3463,DISK] 2024-11-13T03:32:26,592 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468746576 2024-11-13T03:32:26,594 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40391:40391),(127.0.0.1/127.0.0.1:41907:41907),(127.0.0.1/127.0.0.1:34497:34497)] 2024-11-13T03:32:26,594 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-13T03:32:26,596 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,597 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName a 2024-11-13T03:32:26,597 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:26,602 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/f6e21c796a364889af5a154136520b90 2024-11-13T03:32:26,602 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:26,602 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,603 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName b 2024-11-13T03:32:26,603 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:26,607 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/b/5b12e8c4c023438abfbe239f01f28db0 2024-11-13T03:32:26,607 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:26,608 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,608 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 62493399fe54f1723fc2dc03d994d24a columnFamilyName c 2024-11-13T03:32:26,608 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-13T03:32:26,613 DEBUG [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/c/22e59e87182540c8b000a3c710808e07 2024-11-13T03:32:26,613 INFO [StoreOpener-62493399fe54f1723fc2dc03d994d24a-1 {}] regionserver.HStore(327): Store=62493399fe54f1723fc2dc03d994d24a/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-13T03:32:26,613 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,615 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,616 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 2024-11-13T03:32:26,621 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 2024-11-13T03:32:26,622 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 62493399fe54f1723fc2dc03d994d24a 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-11-13T03:32:26,635 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/792cb91646944d708863623f7679e758 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1731468742494/Put/seqid=0 2024-11-13T03:32:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741940_1120 (size=5958) 2024-11-13T03:32:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741940_1120 (size=5958) 2024-11-13T03:32:26,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741940_1120 (size=5958) 2024-11-13T03:32:26,642 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/792cb91646944d708863623f7679e758 2024-11-13T03:32:26,661 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/07c591b88906415abd5aa1098871988f is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1731468742500/Put/seqid=0 2024-11-13T03:32:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741941_1121 (size=5958) 2024-11-13T03:32:26,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741941_1121 (size=5958) 2024-11-13T03:32:26,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741941_1121 (size=5958) 2024-11-13T03:32:26,667 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/07c591b88906415abd5aa1098871988f 2024-11-13T03:32:26,685 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/0aafe2c6f3674ed083fb9353464d67ed is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1731468742505/Put/seqid=0 2024-11-13T03:32:26,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741942_1122 (size=5958) 2024-11-13T03:32:26,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741942_1122 (size=5958) 2024-11-13T03:32:26,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741942_1122 (size=5958) 2024-11-13T03:32:26,691 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/0aafe2c6f3674ed083fb9353464d67ed 2024-11-13T03:32:26,695 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/a/792cb91646944d708863623f7679e758 as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/792cb91646944d708863623f7679e758 2024-11-13T03:32:26,699 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/a/792cb91646944d708863623f7679e758, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:32:26,700 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/b/07c591b88906415abd5aa1098871988f as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/b/07c591b88906415abd5aa1098871988f 2024-11-13T03:32:26,705 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/b/07c591b88906415abd5aa1098871988f, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:32:26,706 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/.tmp/c/0aafe2c6f3674ed083fb9353464d67ed as hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/c/0aafe2c6f3674ed083fb9353464d67ed 2024-11-13T03:32:26,710 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/c/0aafe2c6f3674ed083fb9353464d67ed, entries=10, sequenceid=66, filesize=5.8 K 2024-11-13T03:32:26,710 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 62493399fe54f1723fc2dc03d994d24a in 89ms, sequenceid=66, compaction requested=false; wal=null 2024-11-13T03:32:26,711 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/0000000000000000066 2024-11-13T03:32:26,712 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,712 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,712 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-13T03:32:26,714 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 62493399fe54f1723fc2dc03d994d24a 2024-11-13T03:32:26,716 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/hbase/data/default/testReplayEditsWrittenViaHRegion/62493399fe54f1723fc2dc03d994d24a/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-11-13T03:32:26,716 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 62493399fe54f1723fc2dc03d994d24a; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73578904, jitterRate=0.09641110897064209}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-13T03:32:26,716 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 62493399fe54f1723fc2dc03d994d24a: Writing region info on filesystem at 1731468746595Initializing all the Stores at 1731468746595Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468746596 (+1 ms)Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468746596Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731468746596Obtaining lock to block concurrent updates at 1731468746622 (+26 ms)Preparing flush snapshotting stores in 62493399fe54f1723fc2dc03d994d24a at 1731468746622Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1731468746622Flushing stores of testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. at 1731468746622Flushing 62493399fe54f1723fc2dc03d994d24a/a: creating writer at 1731468746622Flushing 62493399fe54f1723fc2dc03d994d24a/a: appending metadata at 1731468746635 (+13 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/a: closing flushed file at 1731468746635Flushing 62493399fe54f1723fc2dc03d994d24a/b: creating writer at 1731468746646 (+11 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/b: appending metadata at 1731468746660 (+14 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/b: closing flushed file at 1731468746660Flushing 62493399fe54f1723fc2dc03d994d24a/c: creating writer at 1731468746671 (+11 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/c: appending metadata at 1731468746684 (+13 ms)Flushing 62493399fe54f1723fc2dc03d994d24a/c: closing flushed file at 1731468746684Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@208d0678: reopening flushed file at 1731468746695 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@578b3915: reopening flushed file at 1731468746700 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e8ad908: reopening flushed file at 1731468746705 (+5 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 62493399fe54f1723fc2dc03d994d24a in 89ms, sequenceid=66, compaction requested=false; wal=null at 1731468746710 (+5 ms)Cleaning up temporary data from old regions at 1731468746712 (+2 ms)Region opened successfully at 1731468746716 (+4 ms) 2024-11-13T03:32:26,727 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 62493399fe54f1723fc2dc03d994d24a, disabling compactions & flushes 2024-11-13T03:32:26,727 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:26,727 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:26,727 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. after waiting 0 ms 2024-11-13T03:32:26,727 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:26,728 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1731468742179.62493399fe54f1723fc2dc03d994d24a. 2024-11-13T03:32:26,729 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 62493399fe54f1723fc2dc03d994d24a: Waiting for close lock at 1731468746727Disabling compacts and flushes for region at 1731468746727Disabling writes for close at 1731468746727Writing region close event to WAL at 1731468746728 (+1 ms)Closed at 1731468746728 2024-11-13T03:32:26,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741939_1119 (size=93) 2024-11-13T03:32:26,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741939_1119 (size=93) 2024-11-13T03:32:26,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741939_1119 (size=93) 2024-11-13T03:32:26,733 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-11-13T03:32:26,733 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1731468746576) 2024-11-13T03:32:26,745 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=449 (was 443) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1137707606_22 at /127.0.0.1:59040 [Waiting for operation #28] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1137707606_22 at /127.0.0.1:35762 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:33705 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1382578499) connection to localhost/127.0.0.1:33705 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1137707606_22 at /127.0.0.1:33726 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1414 (was 1348) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=227 (was 246), ProcessCount=11 (was 11), AvailableMemoryMB=2727 (was 2737) 2024-11-13T03:32:26,745 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1414 is superior to 1024 2024-11-13T03:32:26,745 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-13T03:32:26,745 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:32:26,745 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:26,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:26,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:26,746 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-13T03:32:26,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-13T03:32:26,746 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=655895052, stopped=false 2024-11-13T03:32:26,747 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=8394b182524b,41967,1731468706868 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:26,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:26,808 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:32:26,808 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-13T03:32:26,809 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:26,809 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:26,809 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,43461,1731468707627' ***** 2024-11-13T03:32:26,809 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T03:32:26,809 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '8394b182524b,41763,1731468707723' ***** 2024-11-13T03:32:26,809 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:26,809 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-13T03:32:26,809 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:26,809 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-13T03:32:26,809 INFO [RS:1;8394b182524b:41763 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:32:26,809 INFO [RS:1;8394b182524b:41763 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T03:32:26,809 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:32:26,809 INFO [RS:1;8394b182524b:41763 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-13T03:32:26,810 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(3091): Received CLOSE for fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(959): stopping server 8394b182524b,43461,1731468707627 2024-11-13T03:32:26,810 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;8394b182524b:43461. 2024-11-13T03:32:26,810 DEBUG [RS:0;8394b182524b:43461 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:26,810 DEBUG [RS:0;8394b182524b:43461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:26,810 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(959): stopping server 8394b182524b,41763,1731468707723 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:32:26,810 INFO [RS:1;8394b182524b:41763 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:32:26,810 INFO [RS:1;8394b182524b:41763 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;8394b182524b:41763. 2024-11-13T03:32:26,810 DEBUG [RS:1;8394b182524b:41763 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-13T03:32:26,810 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-13T03:32:26,810 DEBUG [RS:1;8394b182524b:41763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:26,810 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing fa8df6bead70f7386f753fb4aa9c8270, disabling compactions & flushes 2024-11-13T03:32:26,811 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:26,811 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:32:26,811 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1325): Online Regions={fa8df6bead70f7386f753fb4aa9c8270=testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270.} 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:26,811 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-13T03:32:26,811 DEBUG [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1351): Waiting on fa8df6bead70f7386f753fb4aa9c8270 2024-11-13T03:32:26,811 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. after waiting 0 ms 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:26,811 DEBUG [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-13T03:32:26,811 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-13T03:32:26,811 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-13T03:32:26,811 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.19 KB heapSize=2.79 KB 2024-11-13T03:32:26,814 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/default/testReplayEditsAfterRegionMovedWithMultiCF/fa8df6bead70f7386f753fb4aa9c8270/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-11-13T03:32:26,815 INFO [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:26,815 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for fa8df6bead70f7386f753fb4aa9c8270: Waiting for close lock at 1731468746810Running coprocessor pre-close hooks at 1731468746810Disabling compacts and flushes for region at 1731468746810Disabling writes for close at 1731468746811 (+1 ms)Writing region close event to WAL at 1731468746811Running coprocessor post-close hooks at 1731468746815 (+4 ms)Closed at 1731468746815 2024-11-13T03:32:26,815 DEBUG [RS_CLOSE_REGION-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270. 2024-11-13T03:32:26,815 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/138341dfe1dc46beb81c23948a0a4f48 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1731468722010.fa8df6bead70f7386f753fb4aa9c8270./info:regioninfo/1731468726104/Put/seqid=0 2024-11-13T03:32:26,818 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:26,818 INFO [regionserver/8394b182524b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741943_1123 (size=6778) 2024-11-13T03:32:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741943_1123 (size=6778) 2024-11-13T03:32:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741943_1123 (size=6778) 2024-11-13T03:32:26,822 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.19 KB at sequenceid=23 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/138341dfe1dc46beb81c23948a0a4f48 2024-11-13T03:32:26,829 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/.tmp/info/138341dfe1dc46beb81c23948a0a4f48 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/info/138341dfe1dc46beb81c23948a0a4f48 2024-11-13T03:32:26,835 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/info/138341dfe1dc46beb81c23948a0a4f48, entries=8, sequenceid=23, filesize=6.6 K 2024-11-13T03:32:26,836 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false 2024-11-13T03:32:26,836 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-13T03:32:26,841 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/data/hbase/meta/1588230740/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=18 2024-11-13T03:32:26,842 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:32:26,842 INFO [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-13T03:32:26,842 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731468746811Running coprocessor pre-close hooks at 1731468746811Disabling compacts and flushes for region at 1731468746811Disabling writes for close at 1731468746811Obtaining lock to block concurrent updates at 1731468746811Preparing flush snapshotting stores in 1588230740 at 1731468746811Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1218, getHeapSize=2792, getOffHeapSize=0, getCellsCount=8 at 1731468746811Flushing stores of hbase:meta,,1.1588230740 at 1731468746812 (+1 ms)Flushing 1588230740/info: creating writer at 1731468746812Flushing 1588230740/info: appending metadata at 1731468746815 (+3 ms)Flushing 1588230740/info: closing flushed file at 1731468746815Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@319671de: reopening flushed file at 1731468746828 (+13 ms)Finished flush of dataSize ~1.19 KB/1218, heapSize ~2.02 KB/2072, currentSize=0 B/0 for 1588230740 in 25ms, sequenceid=23, compaction requested=false at 1731468746836 (+8 ms)Writing region close event to WAL at 1731468746838 (+2 ms)Running coprocessor post-close hooks at 1731468746841 (+3 ms)Closed at 1731468746842 (+1 ms) 2024-11-13T03:32:26,842 DEBUG [RS_CLOSE_META-regionserver/8394b182524b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-13T03:32:27,011 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,41763,1731468707723; all regions closed. 2024-11-13T03:32:27,011 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(976): stopping server 8394b182524b,43461,1731468707627; all regions closed. 2024-11-13T03:32:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741833_1009 (size=928) 2024-11-13T03:32:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741833_1009 (size=928) 2024-11-13T03:32:27,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741833_1009 (size=928) 2024-11-13T03:32:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741893_1071 (size=2422) 2024-11-13T03:32:27,016 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/WALs/8394b182524b,43461,1731468707627/8394b182524b%2C43461%2C1731468707627.meta.1731468725262.meta not finished, retry = 0 2024-11-13T03:32:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741893_1071 (size=2422) 2024-11-13T03:32:27,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741893_1071 (size=2422) 2024-11-13T03:32:27,018 DEBUG [RS:1;8394b182524b:41763 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs 2024-11-13T03:32:27,018 INFO [RS:1;8394b182524b:41763 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C41763%2C1731468707723:(num 1731468709325) 2024-11-13T03:32:27,018 DEBUG [RS:1;8394b182524b:41763 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:27,018 INFO [RS:1;8394b182524b:41763 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:27,018 INFO [RS:1;8394b182524b:41763 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-13T03:32:27,019 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:32:27,019 INFO [RS:1;8394b182524b:41763 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41763 2024-11-13T03:32:27,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,41763,1731468707723 2024-11-13T03:32:27,084 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:32:27,084 INFO [RS:1;8394b182524b:41763 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:32:27,103 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,41763,1731468707723] 2024-11-13T03:32:27,113 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,41763,1731468707723 already deleted, retry=false 2024-11-13T03:32:27,114 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8394b182524b,41763,1731468707723 expired; onlineServers=1 2024-11-13T03:32:27,123 DEBUG [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs 2024-11-13T03:32:27,123 INFO [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C43461%2C1731468707627.meta:.meta(num 1731468725262) 2024-11-13T03:32:27,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741834_1010 (size=2598) 2024-11-13T03:32:27,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741834_1010 (size=2598) 2024-11-13T03:32:27,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741834_1010 (size=2598) 2024-11-13T03:32:27,131 DEBUG [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/oldWALs 2024-11-13T03:32:27,131 INFO [RS:0;8394b182524b:43461 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 8394b182524b%2C43461%2C1731468707627:(num 1731468709325) 2024-11-13T03:32:27,131 DEBUG [RS:0;8394b182524b:43461 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-13T03:32:27,131 INFO [RS:0;8394b182524b:43461 {}] regionserver.LeaseManager(133): Closed leases 2024-11-13T03:32:27,131 INFO [RS:0;8394b182524b:43461 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:32:27,131 INFO [RS:0;8394b182524b:43461 {}] hbase.ChoreService(370): Chore service for: regionserver/8394b182524b:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-13T03:32:27,131 INFO [RS:0;8394b182524b:43461 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:32:27,131 INFO [regionserver/8394b182524b:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:32:27,132 INFO [RS:0;8394b182524b:43461 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:43461 2024-11-13T03:32:27,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-13T03:32:27,145 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/8394b182524b,43461,1731468707627 2024-11-13T03:32:27,145 INFO [RS:0;8394b182524b:43461 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:32:27,146 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [8394b182524b,43461,1731468707627] 2024-11-13T03:32:27,166 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/8394b182524b,43461,1731468707627 already deleted, retry=false 2024-11-13T03:32:27,166 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 8394b182524b,43461,1731468707627 expired; onlineServers=0 2024-11-13T03:32:27,166 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '8394b182524b,41967,1731468706868' ***** 2024-11-13T03:32:27,166 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-13T03:32:27,166 INFO [M:0;8394b182524b:41967 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-13T03:32:27,166 INFO [M:0;8394b182524b:41967 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-13T03:32:27,166 DEBUG [M:0;8394b182524b:41967 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-13T03:32:27,167 DEBUG [M:0;8394b182524b:41967 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-13T03:32:27,167 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-13T03:32:27,167 DEBUG [master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468709002 {}] cleaner.HFileCleaner(306): Exit Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.large.0-1731468709002,5,FailOnTimeoutGroup] 2024-11-13T03:32:27,167 DEBUG [master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468709003 {}] cleaner.HFileCleaner(306): Exit Thread[master/8394b182524b:0:becomeActiveMaster-HFileCleaner.small.0-1731468709003,5,FailOnTimeoutGroup] 2024-11-13T03:32:27,167 INFO [M:0;8394b182524b:41967 {}] hbase.ChoreService(370): Chore service for: master/8394b182524b:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-13T03:32:27,167 INFO [M:0;8394b182524b:41967 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-13T03:32:27,167 DEBUG [M:0;8394b182524b:41967 {}] master.HMaster(1795): Stopping service threads 2024-11-13T03:32:27,167 INFO [M:0;8394b182524b:41967 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-13T03:32:27,167 INFO [M:0;8394b182524b:41967 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-13T03:32:27,168 INFO [M:0;8394b182524b:41967 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-13T03:32:27,169 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-13T03:32:27,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-13T03:32:27,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-13T03:32:27,177 DEBUG [M:0;8394b182524b:41967 {}] zookeeper.ZKUtil(347): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-13T03:32:27,177 WARN [M:0;8394b182524b:41967 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-13T03:32:27,178 INFO [M:0;8394b182524b:41967 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/.lastflushedseqids 2024-11-13T03:32:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741944_1124 (size=119) 2024-11-13T03:32:27,194 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741944_1124 (size=119) 2024-11-13T03:32:27,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741944_1124 (size=119) 2024-11-13T03:32:27,195 INFO [M:0;8394b182524b:41967 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-13T03:32:27,195 INFO [M:0;8394b182524b:41967 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-13T03:32:27,195 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-13T03:32:27,195 INFO [M:0;8394b182524b:41967 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:27,195 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:27,195 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-13T03:32:27,195 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:27,195 INFO [M:0;8394b182524b:41967 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=83.40 KB heapSize=102.70 KB 2024-11-13T03:32:27,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,203 INFO [RS:1;8394b182524b:41763 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:32:27,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41763-0x101323fbcd90002, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,203 INFO [RS:1;8394b182524b:41763 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,41763,1731468707723; zookeeper connection closed. 2024-11-13T03:32:27,203 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@27494e79 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@27494e79 2024-11-13T03:32:27,210 DEBUG [M:0;8394b182524b:41967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd87612c54394cfd89f3a537a4f88ae6 is 82, key is hbase:meta,,1/info:regioninfo/1731468725407/Put/seqid=0 2024-11-13T03:32:27,214 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-11-13T03:32:27,214 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:27,215 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-11-13T03:32:27,215 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-11-13T03:32:27,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741945_1125 (size=6063) 2024-11-13T03:32:27,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741945_1125 (size=6063) 2024-11-13T03:32:27,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-13T03:32:27,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741945_1125 (size=6063) 2024-11-13T03:32:27,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-13T03:32:27,216 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-11-13T03:32:27,216 INFO [M:0;8394b182524b:41967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1008 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd87612c54394cfd89f3a537a4f88ae6 2024-11-13T03:32:27,235 DEBUG [M:0;8394b182524b:41967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330c3f08556b46c19552e61add6e7235 is 1076, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1731468722429/Put/seqid=0 2024-11-13T03:32:27,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741946_1126 (size=7907) 2024-11-13T03:32:27,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741946_1126 (size=7907) 2024-11-13T03:32:27,242 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741946_1126 (size=7907) 2024-11-13T03:32:27,242 INFO [M:0;8394b182524b:41967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=82.17 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330c3f08556b46c19552e61add6e7235 2024-11-13T03:32:27,247 INFO [M:0;8394b182524b:41967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 330c3f08556b46c19552e61add6e7235 2024-11-13T03:32:27,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,256 INFO [RS:0;8394b182524b:43461 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:32:27,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:43461-0x101323fbcd90001, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,256 INFO [RS:0;8394b182524b:43461 {}] regionserver.HRegionServer(1031): Exiting; stopping=8394b182524b,43461,1731468707627; zookeeper connection closed. 2024-11-13T03:32:27,256 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@795eab0c {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@795eab0c 2024-11-13T03:32:27,256 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-11-13T03:32:27,268 DEBUG [M:0;8394b182524b:41967 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fe598d13a8e4e7b8f8aaf1d233c974a is 69, key is 8394b182524b,41763,1731468707723/rs:state/1731468709044/Put/seqid=0 2024-11-13T03:32:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741947_1127 (size=5440) 2024-11-13T03:32:27,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741947_1127 (size=5440) 2024-11-13T03:32:27,274 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741947_1127 (size=5440) 2024-11-13T03:32:27,274 INFO [M:0;8394b182524b:41967 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fe598d13a8e4e7b8f8aaf1d233c974a 2024-11-13T03:32:27,278 INFO [M:0;8394b182524b:41967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fe598d13a8e4e7b8f8aaf1d233c974a 2024-11-13T03:32:27,279 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/bd87612c54394cfd89f3a537a4f88ae6 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd87612c54394cfd89f3a537a4f88ae6 2024-11-13T03:32:27,284 INFO [M:0;8394b182524b:41967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/bd87612c54394cfd89f3a537a4f88ae6, entries=14, sequenceid=207, filesize=5.9 K 2024-11-13T03:32:27,285 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/330c3f08556b46c19552e61add6e7235 as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/330c3f08556b46c19552e61add6e7235 2024-11-13T03:32:27,290 INFO [M:0;8394b182524b:41967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 330c3f08556b46c19552e61add6e7235 2024-11-13T03:32:27,290 INFO [M:0;8394b182524b:41967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/330c3f08556b46c19552e61add6e7235, entries=21, sequenceid=207, filesize=7.7 K 2024-11-13T03:32:27,291 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/6fe598d13a8e4e7b8f8aaf1d233c974a as hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6fe598d13a8e4e7b8f8aaf1d233c974a 2024-11-13T03:32:27,295 INFO [M:0;8394b182524b:41967 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 6fe598d13a8e4e7b8f8aaf1d233c974a 2024-11-13T03:32:27,295 INFO [M:0;8394b182524b:41967 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:33705/user/jenkins/test-data/a2e0b335-66ff-db47-3001-fdcdffd4ccda/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/6fe598d13a8e4e7b8f8aaf1d233c974a, entries=3, sequenceid=207, filesize=5.3 K 2024-11-13T03:32:27,296 INFO [M:0;8394b182524b:41967 {}] regionserver.HRegion(3140): Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=207, compaction requested=false 2024-11-13T03:32:27,297 INFO [M:0;8394b182524b:41967 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-13T03:32:27,297 DEBUG [M:0;8394b182524b:41967 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731468747195Disabling compacts and flushes for region at 1731468747195Disabling writes for close at 1731468747195Obtaining lock to block concurrent updates at 1731468747195Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731468747195Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=85398, getHeapSize=105104, getOffHeapSize=0, getCellsCount=248 at 1731468747195Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731468747196 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731468747196Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731468747210 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731468747210Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731468747220 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731468747234 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731468747234Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731468747247 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731468747267 (+20 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731468747267Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@689f8ba4: reopening flushed file at 1731468747278 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@14bc1132: reopening flushed file at 1731468747284 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@164d025a: reopening flushed file at 1731468747290 (+6 ms)Finished flush of dataSize ~83.40 KB/85398, heapSize ~102.41 KB/104864, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 101ms, sequenceid=207, compaction requested=false at 1731468747296 (+6 ms)Writing region close event to WAL at 1731468747297 (+1 ms)Closed at 1731468747297 2024-11-13T03:32:27,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37801 is added to blk_1073741830_1006 (size=99436) 2024-11-13T03:32:27,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43285 is added to blk_1073741830_1006 (size=99436) 2024-11-13T03:32:27,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43451 is added to blk_1073741830_1006 (size=99436) 2024-11-13T03:32:27,300 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-13T03:32:27,300 INFO [M:0;8394b182524b:41967 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-13T03:32:27,300 INFO [M:0;8394b182524b:41967 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:41967 2024-11-13T03:32:27,301 INFO [M:0;8394b182524b:41967 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-13T03:32:27,413 INFO [M:0;8394b182524b:41967 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-13T03:32:27,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,413 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:41967-0x101323fbcd90000, quorum=127.0.0.1:58534, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-13T03:32:27,419 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468736732 with renewLeaseKey: DEFAULT_16688 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468736732 (inode 16688) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1731468727493/wal.1731468736732 (inode 16688) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:32:27,422 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727328 with renewLeaseKey: DEFAULT_16665 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727328 (inode 16665) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1731468726642/wal.1731468727328 (inode 16665) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:32:27,423 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1731468717522/wal.1731468717714 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:27,425 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737502 with renewLeaseKey: DEFAULT_16714 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737502 (inode 16714) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1731468736979/wal.1731468737502 (inode 16714) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:32:27,425 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1731468737650/wal.1731468737727 with renewLeaseKey: DEFAULT_16736 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:27,425 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1731468742178/wal.1731468742383 with renewLeaseKey: DEFAULT_16777 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:27,426 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1731468711102/wal.1731468711172 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-13T03:32:27,427 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal.1731468710968 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal.1731468710968 (inode 16485) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1731468710778/wal.1731468710968 (inode 16485) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:32:27,429 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal.1731468710545 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal.1731468710545 (inode 16462) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1731468710323/wal.1731468710545 (inode 16462) Holder DFSClient_NONMAPREDUCE_1314386333_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-11-13T03:32:27,433 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@3217543c{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:27,435 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@13cfb6c5{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:32:27,435 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:32:27,436 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6b2b98f9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:32:27,436 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@22fb7089{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,STOPPED} 2024-11-13T03:32:27,438 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:32:27,438 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:32:27,438 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-648661827-172.17.0.3-1731468702443 (Datanode Uuid b4309cbc-40ad-4408-a07a-b48671724b7a) service to localhost/127.0.0.1:33705 2024-11-13T03:32:27,438 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:32:27,439 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data5/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,439 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data6/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,439 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:32:27,441 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@7e9308b2{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:27,442 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4f18ac21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:32:27,442 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:32:27,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@918952d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:32:27,442 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a707162{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,STOPPED} 2024-11-13T03:32:27,443 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:32:27,443 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:32:27,443 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-648661827-172.17.0.3-1731468702443 (Datanode Uuid a477af8c-4f94-4dc1-a388-b8b6c03e8704) service to localhost/127.0.0.1:33705 2024-11-13T03:32:27,443 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:32:27,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data3/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,444 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data4/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,444 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:32:27,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@54387323{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-13T03:32:27,453 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7a45a21{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:32:27,453 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:32:27,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7ce178fc{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:32:27,453 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@75929b0a{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,STOPPED} 2024-11-13T03:32:27,454 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-13T03:32:27,454 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-13T03:32:27,454 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-13T03:32:27,455 WARN [BP-648661827-172.17.0.3-1731468702443 heartbeating to localhost/127.0.0.1:33705 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-648661827-172.17.0.3-1731468702443 (Datanode Uuid 01ac62a3-65ab-469f-ad86-8fba4fe12067) service to localhost/127.0.0.1:33705 2024-11-13T03:32:27,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data2/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,456 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/cluster_8260819d-2835-e1d0-5952-764ca689615d/data/data1/current/BP-648661827-172.17.0.3-1731468702443 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-13T03:32:27,456 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-13T03:32:27,464 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@398d2d49{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-13T03:32:27,464 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@166111eb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-13T03:32:27,465 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-13T03:32:27,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71d2272d{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-13T03:32:27,465 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6d4d6e89{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/f1c3ee84-64bb-78c8-80c4-da3948c4654a/hadoop.log.dir/,STOPPED} 2024-11-13T03:32:27,473 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-13T03:32:27,525 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down