2024-12-03 05:25:25,423 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 05:25:25,433 main DEBUG Took 0.008382 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-03 05:25:25,433 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-03 05:25:25,433 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-03 05:25:25,434 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-03 05:25:25,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,441 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-03 05:25:25,452 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,454 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,454 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,455 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,456 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,457 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,458 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,458 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,459 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,459 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,460 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-03 05:25:25,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,461 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-03 05:25:25,463 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-03 05:25:25,464 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-03 05:25:25,465 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-03 05:25:25,466 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-03 05:25:25,467 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-03 05:25:25,467 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-03 05:25:25,475 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-03 05:25:25,477 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-03 05:25:25,479 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-03 05:25:25,479 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-03 05:25:25,479 main DEBUG createAppenders(={Console}) 2024-12-03 05:25:25,480 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a initialized 2024-12-03 05:25:25,481 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a 2024-12-03 05:25:25,481 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@20b2475a OK. 2024-12-03 05:25:25,481 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-03 05:25:25,482 main DEBUG OutputStream closed 2024-12-03 05:25:25,482 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-03 05:25:25,482 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-03 05:25:25,482 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4310d43 OK 2024-12-03 05:25:25,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-03 05:25:25,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-03 05:25:25,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-03 05:25:25,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-03 05:25:25,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-03 05:25:25,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-03 05:25:25,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-03 05:25:25,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-03 05:25:25,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-03 05:25:25,568 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-03 05:25:25,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-03 05:25:25,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-03 05:25:25,569 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-03 05:25:25,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-03 05:25:25,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-03 05:25:25,570 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-03 05:25:25,571 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-03 05:25:25,572 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-03 05:25:25,575 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03 05:25:25,581 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@12f9af83) with optional ClassLoader: null 2024-12-03 05:25:25,581 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-03 05:25:25,582 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@12f9af83] started OK. 2024-12-03T05:25:25,832 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085 2024-12-03 05:25:25,834 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-03 05:25:25,834 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-03T05:25:25,843 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay timeout: 13 mins 2024-12-03T05:25:25,850 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.regionserver.wal.TestSecureAsyncWALReplay timeout: 13 mins 2024-12-03T05:25:25,872 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-03T05:25:25,927 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-03T05:25:25,927 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-03T05:25:25,942 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=3, rsPorts=, rsClass=null, numDataNodes=3, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-03T05:25:25,957 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87, deleteOnExit=true 2024-12-03T05:25:25,957 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-03T05:25:25,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/test.cache.data in system properties and HBase conf 2024-12-03T05:25:25,958 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.tmp.dir in system properties and HBase conf 2024-12-03T05:25:25,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir in system properties and HBase conf 2024-12-03T05:25:25,959 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-03T05:25:25,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-03T05:25:25,960 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-03T05:25:26,059 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-03T05:25:26,176 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-03T05:25:26,180 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-03T05:25:26,181 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-03T05:25:26,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-03T05:25:26,182 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T05:25:26,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-03T05:25:26,183 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-03T05:25:26,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-03T05:25:26,184 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T05:25:26,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-03T05:25:26,185 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/nfs.dump.dir in system properties and HBase conf 2024-12-03T05:25:26,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/java.io.tmpdir in system properties and HBase conf 2024-12-03T05:25:26,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-03T05:25:26,186 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-03T05:25:26,187 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-03T05:25:27,652 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-03T05:25:27,779 INFO [Time-limited test {}] log.Log(170): Logging initialized @3017ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-03T05:25:27,890 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:25:27,995 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:25:28,129 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:25:28,130 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:25:28,132 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T05:25:28,162 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:25:28,171 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:25:28,172 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:25:28,630 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4c03e044{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/java.io.tmpdir/jetty-localhost-35971-hadoop-hdfs-3_4_1-tests_jar-_-any-13131125158369731725/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T05:25:28,654 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:35971} 2024-12-03T05:25:28,656 INFO [Time-limited test {}] server.Server(415): Started @3893ms 2024-12-03T05:25:29,623 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:25:29,631 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:25:29,634 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:25:29,634 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:25:29,634 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-03T05:25:29,637 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:25:29,638 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:25:29,785 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@501095cc{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/java.io.tmpdir/jetty-localhost-43255-hadoop-hdfs-3_4_1-tests_jar-_-any-14725541420706838832/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:25:29,788 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:43255} 2024-12-03T05:25:29,789 INFO [Time-limited test {}] server.Server(415): Started @5027ms 2024-12-03T05:25:29,855 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T05:25:30,080 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:25:30,087 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:25:30,093 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:25:30,093 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:25:30,093 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T05:25:30,096 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:25:30,097 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:25:30,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@64052af6{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/java.io.tmpdir/jetty-localhost-37877-hadoop-hdfs-3_4_1-tests_jar-_-any-2762658157856370681/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:25:30,247 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:37877} 2024-12-03T05:25:30,247 INFO [Time-limited test {}] server.Server(415): Started @5485ms 2024-12-03T05:25:30,259 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T05:25:30,335 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-03T05:25:30,341 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-03T05:25:30,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-03T05:25:30,361 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-03T05:25:30,361 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-03T05:25:30,365 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,AVAILABLE} 2024-12-03T05:25:30,365 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-03T05:25:30,515 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@420bc738{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/java.io.tmpdir/jetty-localhost-43879-hadoop-hdfs-3_4_1-tests_jar-_-any-3425096342085286178/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:25:30,516 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:43879} 2024-12-03T05:25:30,516 INFO [Time-limited test {}] server.Server(415): Started @5754ms 2024-12-03T05:25:30,519 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-03T05:25:31,978 WARN [Thread-125 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data2/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:31,978 WARN [Thread-122 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data3/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:31,978 WARN [Thread-124 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data1/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:31,978 WARN [Thread-123 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data4/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:32,029 WARN [Thread-81 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T05:25:32,029 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T05:25:32,086 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a0c6ad187d907a6 with lease ID 0x5977f48a1be7a3da: Processing first storage report for DS-9e993d4a-0693-49fe-9591-cfc04dc01190 from datanode DatanodeRegistration(127.0.0.1:45403, datanodeUuid=559ab081-2cbf-42fa-87fa-76b5b8954585, infoPort=42667, infoSecurePort=0, ipcPort=37539, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,086 WARN [Thread-143 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data6/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:32,087 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a0c6ad187d907a6 with lease ID 0x5977f48a1be7a3da: from storage DS-9e993d4a-0693-49fe-9591-cfc04dc01190 node DatanodeRegistration(127.0.0.1:45403, datanodeUuid=559ab081-2cbf-42fa-87fa-76b5b8954585, infoPort=42667, infoSecurePort=0, ipcPort=37539, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,088 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80797f778a818fca with lease ID 0x5977f48a1be7a3d9: Processing first storage report for DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa from datanode DatanodeRegistration(127.0.0.1:40897, datanodeUuid=95d316af-ec37-443a-97a2-cfb6e6f154f4, infoPort=36237, infoSecurePort=0, ipcPort=38479, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,088 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80797f778a818fca with lease ID 0x5977f48a1be7a3d9: from storage DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa node DatanodeRegistration(127.0.0.1:40897, datanodeUuid=95d316af-ec37-443a-97a2-cfb6e6f154f4, infoPort=36237, infoSecurePort=0, ipcPort=38479, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,088 WARN [Thread-142 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data5/current/BP-1175346662-172.17.0.3-1733203526961/current, will proceed with Du for space computation calculation, 2024-12-03T05:25:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x80797f778a818fca with lease ID 0x5977f48a1be7a3d9: Processing first storage report for DS-ca0b2fd1-4dbd-4b04-8689-63832f620817 from datanode DatanodeRegistration(127.0.0.1:40897, datanodeUuid=95d316af-ec37-443a-97a2-cfb6e6f154f4, infoPort=36237, infoSecurePort=0, ipcPort=38479, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x80797f778a818fca with lease ID 0x5977f48a1be7a3d9: from storage DS-ca0b2fd1-4dbd-4b04-8689-63832f620817 node DatanodeRegistration(127.0.0.1:40897, datanodeUuid=95d316af-ec37-443a-97a2-cfb6e6f154f4, infoPort=36237, infoSecurePort=0, ipcPort=38479, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x4a0c6ad187d907a6 with lease ID 0x5977f48a1be7a3da: Processing first storage report for DS-4a66957e-5eb8-42eb-b9fa-922cd7268dc6 from datanode DatanodeRegistration(127.0.0.1:45403, datanodeUuid=559ab081-2cbf-42fa-87fa-76b5b8954585, infoPort=42667, infoSecurePort=0, ipcPort=37539, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,089 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x4a0c6ad187d907a6 with lease ID 0x5977f48a1be7a3da: from storage DS-4a66957e-5eb8-42eb-b9fa-922cd7268dc6 node DatanodeRegistration(127.0.0.1:45403, datanodeUuid=559ab081-2cbf-42fa-87fa-76b5b8954585, infoPort=42667, infoSecurePort=0, ipcPort=37539, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,146 WARN [Thread-103 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-03T05:25:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8867e9977128e4f6 with lease ID 0x5977f48a1be7a3db: Processing first storage report for DS-f92c31f8-5241-4971-b7df-9d8f22bab352 from datanode DatanodeRegistration(127.0.0.1:45109, datanodeUuid=627416e0-b63c-4031-8f58-2014fcb13715, infoPort=41549, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8867e9977128e4f6 with lease ID 0x5977f48a1be7a3db: from storage DS-f92c31f8-5241-4971-b7df-9d8f22bab352 node DatanodeRegistration(127.0.0.1:45109, datanodeUuid=627416e0-b63c-4031-8f58-2014fcb13715, infoPort=41549, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: true, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8867e9977128e4f6 with lease ID 0x5977f48a1be7a3db: Processing first storage report for DS-e67cb62a-d5b7-4386-a8bd-e02e4661d57a from datanode DatanodeRegistration(127.0.0.1:45109, datanodeUuid=627416e0-b63c-4031-8f58-2014fcb13715, infoPort=41549, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961) 2024-12-03T05:25:32,152 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8867e9977128e4f6 with lease ID 0x5977f48a1be7a3db: from storage DS-e67cb62a-d5b7-4386-a8bd-e02e4661d57a node DatanodeRegistration(127.0.0.1:45109, datanodeUuid=627416e0-b63c-4031-8f58-2014fcb13715, infoPort=41549, infoSecurePort=0, ipcPort=39363, storageInfo=lv=-57;cid=testClusterID;nsid=751456468;c=1733203526961), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-03T05:25:32,258 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085 2024-12-03T05:25:32,381 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/zookeeper_0, clientPort=53301, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-03T05:25:32,399 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53301 2024-12-03T05:25:32,413 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:32,416 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:32,743 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741825_1001 (size=7) 2024-12-03T05:25:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741825_1001 (size=7) 2024-12-03T05:25:32,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741825_1001 (size=7) 2024-12-03T05:25:33,154 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 with version=8 2024-12-03T05:25:33,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/hbase-staging 2024-12-03T05:25:33,540 INFO [Time-limited test {}] client.ConnectionUtils(128): master/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:25:33,553 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:33,554 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:33,560 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:25:33,560 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:33,561 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:25:33,748 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-03T05:25:33,837 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-03T05:25:33,851 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-03T05:25:33,856 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:25:33,889 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 64190 (auto-detected) 2024-12-03T05:25:33,893 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-12-03T05:25:33,924 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:37365 2024-12-03T05:25:33,957 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:37365 connecting to ZooKeeper ensemble=127.0.0.1:53301 2024-12-03T05:25:34,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:373650x0, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:25:34,087 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:37365-0x10199a715d70000 connected 2024-12-03T05:25:34,287 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,291 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,303 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:25:34,307 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298, hbase.cluster.distributed=false 2024-12-03T05:25:34,338 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:25:34,352 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=37365 2024-12-03T05:25:34,354 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=37365 2024-12-03T05:25:34,355 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=37365 2024-12-03T05:25:34,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=37365 2024-12-03T05:25:34,356 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=37365 2024-12-03T05:25:34,499 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:25:34,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,502 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:25:34,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,502 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:25:34,505 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T05:25:34,508 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:25:34,509 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35421 2024-12-03T05:25:34,512 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35421 connecting to ZooKeeper ensemble=127.0.0.1:53301 2024-12-03T05:25:34,513 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,518 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,538 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:354210x0, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:25:34,538 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:354210x0, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:25:34,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35421-0x10199a715d70001 connected 2024-12-03T05:25:34,543 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T05:25:34,556 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T05:25:34,559 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T05:25:34,566 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:25:34,572 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35421 2024-12-03T05:25:34,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35421 2024-12-03T05:25:34,573 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35421 2024-12-03T05:25:34,574 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35421 2024-12-03T05:25:34,574 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35421 2024-12-03T05:25:34,599 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:25:34,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,599 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,600 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:25:34,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,600 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:25:34,601 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T05:25:34,601 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:25:34,604 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:35935 2024-12-03T05:25:34,607 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:35935 connecting to ZooKeeper ensemble=127.0.0.1:53301 2024-12-03T05:25:34,608 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,613 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:359350x0, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:25:34,639 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:359350x0, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:25:34,639 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:35935-0x10199a715d70002 connected 2024-12-03T05:25:34,640 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T05:25:34,641 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T05:25:34,642 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T05:25:34,643 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:25:34,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=35935 2024-12-03T05:25:34,644 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=35935 2024-12-03T05:25:34,648 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=35935 2024-12-03T05:25:34,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=35935 2024-12-03T05:25:34,649 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=35935 2024-12-03T05:25:34,665 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/e3c6570475b6:0 server-side Connection retries=45 2024-12-03T05:25:34,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,665 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,665 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-03T05:25:34,666 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-03T05:25:34,666 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-03T05:25:34,666 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-03T05:25:34,666 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-03T05:25:34,668 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46191 2024-12-03T05:25:34,670 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46191 connecting to ZooKeeper ensemble=127.0.0.1:53301 2024-12-03T05:25:34,671 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,675 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:34,691 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:461910x0, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-03T05:25:34,692 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:461910x0, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:25:34,693 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46191-0x10199a715d70003 connected 2024-12-03T05:25:34,693 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-03T05:25:34,693 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-03T05:25:34,694 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-03T05:25:34,696 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-03T05:25:34,700 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-03T05:25:34,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46191 2024-12-03T05:25:34,701 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46191 2024-12-03T05:25:34,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-03T05:25:34,704 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46191 2024-12-03T05:25:34,725 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e3c6570475b6:37365 2024-12-03T05:25:34,728 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/e3c6570475b6,37365,1733203533327 2024-12-03T05:25:34,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,744 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,745 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,748 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,750 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e3c6570475b6,37365,1733203533327 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:34,790 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:34,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:34,793 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-03T05:25:34,794 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e3c6570475b6,37365,1733203533327 from backup master directory 2024-12-03T05:25:34,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,807 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e3c6570475b6,37365,1733203533327 2024-12-03T05:25:34,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,809 WARN [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:25:34,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-03T05:25:34,809 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e3c6570475b6,37365,1733203533327 2024-12-03T05:25:34,811 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-03T05:25:34,813 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-03T05:25:34,888 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/hbase.id] with ID: a92647ff-8269-4a17-9036-6ed62d50c959 2024-12-03T05:25:34,889 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/.tmp/hbase.id 2024-12-03T05:25:34,923 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741826_1002 (size=42) 2024-12-03T05:25:34,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741826_1002 (size=42) 2024-12-03T05:25:34,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741826_1002 (size=42) 2024-12-03T05:25:34,934 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/.tmp/hbase.id]:[hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/hbase.id] 2024-12-03T05:25:35,007 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-03T05:25:35,013 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-03T05:25:35,035 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 19ms. 2024-12-03T05:25:35,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,053 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,054 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741827_1003 (size=196) 2024-12-03T05:25:35,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741827_1003 (size=196) 2024-12-03T05:25:35,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741827_1003 (size=196) 2024-12-03T05:25:35,096 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:25:35,098 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-03T05:25:35,106 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:35,129 WARN [IPC Server handler 1 on default port 43427 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) For more information, please enable DEBUG log level on org.apache.hadoop.hdfs.server.blockmanagement.BlockPlacementPolicy and org.apache.hadoop.net.NetworkTopology 2024-12-03T05:25:35,130 WARN [IPC Server handler 1 on default port 43427 {}] protocol.BlockStoragePolicy(161): Failed to place enough replicas: expected size is 1 but only 0 storage types can be selected (replication=3, selected=[], unavailable=[DISK], removed=[DISK], policy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}) 2024-12-03T05:25:35,130 WARN [IPC Server handler 1 on default port 43427 {}] blockmanagement.BlockPlacementPolicyDefault(501): Failed to place enough replicas, still in need of 1 to reach 3 (unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]}, newBlock=true) All required storage types are unavailable: unavailableStorages=[DISK], storagePolicy=BlockStoragePolicy{HOT:7, storageTypes=[DISK], creationFallbacks=[], replicationFallbacks=[ARCHIVE]} 2024-12-03T05:25:35,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741828_1004 (size=1189) 2024-12-03T05:25:35,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741828_1004 (size=1189) 2024-12-03T05:25:35,159 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store 2024-12-03T05:25:35,187 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741829_1005 (size=34) 2024-12-03T05:25:35,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741829_1005 (size=34) 2024-12-03T05:25:35,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741829_1005 (size=34) 2024-12-03T05:25:35,195 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-03T05:25:35,198 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:35,199 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T05:25:35,199 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:25:35,199 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:25:35,200 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T05:25:35,201 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:25:35,201 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:25:35,201 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733203535199Disabling compacts and flushes for region at 1733203535199Disabling writes for close at 1733203535201 (+2 ms)Writing region close event to WAL at 1733203535201Closed at 1733203535201 2024-12-03T05:25:35,203 WARN [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/.initializing 2024-12-03T05:25:35,203 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/WALs/e3c6570475b6,37365,1733203533327 2024-12-03T05:25:35,210 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:35,225 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C37365%2C1733203533327, suffix=, logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/WALs/e3c6570475b6,37365,1733203533327, archiveDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/oldWALs, maxLogs=10 2024-12-03T05:25:35,257 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/WALs/e3c6570475b6,37365,1733203533327/e3c6570475b6%2C37365%2C1733203533327.1733203535230, exclude list is [], retry=0 2024-12-03T05:25:35,287 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.initialize(FanOutOneBlockAsyncDFSOutputHelper.java:413) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:472) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper$5.operationComplete(FanOutOneBlockAsyncDFSOutputHelper.java:467) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.NettyFutureUtils.lambda$addListener$0(NettyFutureUtils.java:56) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListener0(DefaultPromise.java:590) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners0(DefaultPromise.java:583) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListenersNow(DefaultPromise.java:559) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.notifyListeners(DefaultPromise.java:492) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setValue0(DefaultPromise.java:636) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.setSuccess0(DefaultPromise.java:625) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.DefaultPromise.trySuccess(DefaultPromise.java:105) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPromise.trySuccess(DefaultChannelPromise.java:84) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.fulfillConnectPromise(AbstractEpollChannel.java:658) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.finishConnect(AbstractEpollChannel.java:696) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollChannel$AbstractEpollUnsafe.epollOutReady(AbstractEpollChannel.java:567) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:491) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:35,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:35,297 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:35,298 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:35,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-03T05:25:35,348 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] crypto.Encryption(565): Installed org.apache.hadoop.hbase.io.crypto.MockAesKeyProvider into key provider cache 2024-12-03T05:25:35,434 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/WALs/e3c6570475b6,37365,1733203533327/e3c6570475b6%2C37365%2C1733203533327.1733203535230 2024-12-03T05:25:35,435 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:35,436 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:35,436 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:35,440 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,441 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,486 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,510 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-03T05:25:35,516 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:35,519 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:35,520 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,525 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-03T05:25:35,525 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:35,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:35,526 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,529 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-03T05:25:35,529 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:35,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:35,530 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,533 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-03T05:25:35,533 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:35,534 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:35,534 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,537 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,538 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,543 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,543 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,546 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T05:25:35,549 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-03T05:25:35,557 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:35,558 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61734897, jitterRate=-0.08007834851741791}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T05:25:35,566 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733203535458Initializing all the Stores at 1733203535461 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203535462 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203535463 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203535463Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203535463Cleaning up temporary data from old regions at 1733203535543 (+80 ms)Region opened successfully at 1733203535566 (+23 ms) 2024-12-03T05:25:35,568 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-03T05:25:35,614 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f3bfb6b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:25:35,654 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-03T05:25:35,667 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-03T05:25:35,667 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-03T05:25:35,670 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-03T05:25:35,672 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-03T05:25:35,678 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 6 msec 2024-12-03T05:25:35,679 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-03T05:25:35,714 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-03T05:25:35,725 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-03T05:25:35,744 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-03T05:25:35,747 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-03T05:25:35,753 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-03T05:25:35,765 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-03T05:25:35,767 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-03T05:25:35,771 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-03T05:25:35,779 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-03T05:25:35,781 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-03T05:25:35,790 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-03T05:25:35,807 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-03T05:25:35,817 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,832 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,833 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,835 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=e3c6570475b6,37365,1733203533327, sessionid=0x10199a715d70000, setting cluster-up flag (Was=false) 2024-12-03T05:25:35,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,895 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-03T05:25:35,897 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3c6570475b6,37365,1733203533327 2024-12-03T05:25:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:35,954 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-03T05:25:35,956 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e3c6570475b6,37365,1733203533327 2024-12-03T05:25:35,962 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-03T05:25:36,013 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(746): ClusterId : a92647ff-8269-4a17-9036-6ed62d50c959 2024-12-03T05:25:36,016 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T05:25:36,019 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(746): ClusterId : a92647ff-8269-4a17-9036-6ed62d50c959 2024-12-03T05:25:36,020 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T05:25:36,039 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(746): ClusterId : a92647ff-8269-4a17-9036-6ed62d50c959 2024-12-03T05:25:36,039 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-03T05:25:36,042 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T05:25:36,042 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T05:25:36,049 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T05:25:36,049 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T05:25:36,060 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-03T05:25:36,071 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-03T05:25:36,079 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-03T05:25:36,086 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e3c6570475b6,37365,1733203533327 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-03T05:25:36,192 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-03T05:25:36,193 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-03T05:25:36,194 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T05:25:36,194 DEBUG [RS:1;e3c6570475b6:35935 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a486, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:25:36,196 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T05:25:36,196 DEBUG [RS:0;e3c6570475b6:35421 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79b9ca91, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:25:36,219 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:1;e3c6570475b6:35935 2024-12-03T05:25:36,219 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e3c6570475b6:35421 2024-12-03T05:25:36,223 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T05:25:36,223 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T05:25:36,223 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T05:25:36,223 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T05:25:36,224 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T05:25:36,224 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T05:25:36,236 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-03T05:25:36,236 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(2659): reportForDuty to master=e3c6570475b6,37365,1733203533327 with port=35935, startcode=1733203534598 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:25:36,237 DEBUG [RS:2;e3c6570475b6:46191 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d778c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e3c6570475b6/172.17.0.3:0 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e3c6570475b6:0, corePoolSize=5, maxPoolSize=5 2024-12-03T05:25:36,237 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(2659): reportForDuty to master=e3c6570475b6,37365,1733203533327 with port=35421, startcode=1733203534454 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e3c6570475b6:0, corePoolSize=10, maxPoolSize=10 2024-12-03T05:25:36,237 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,238 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:25:36,238 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,263 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:2;e3c6570475b6:46191 2024-12-03T05:25:36,263 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-03T05:25:36,263 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-03T05:25:36,264 DEBUG [RS:0;e3c6570475b6:35421 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T05:25:36,264 DEBUG [RS:1;e3c6570475b6:35935 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T05:25:36,265 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-03T05:25:36,266 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(2659): reportForDuty to master=e3c6570475b6,37365,1733203533327 with port=46191, startcode=1733203534664 2024-12-03T05:25:36,267 DEBUG [RS:2;e3c6570475b6:46191 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-03T05:25:36,292 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733203566292 2024-12-03T05:25:36,294 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-03T05:25:36,295 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-03T05:25:36,299 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-03T05:25:36,299 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-03T05:25:36,299 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-03T05:25:36,300 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-03T05:25:36,301 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,310 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:25:36,310 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-03T05:25:36,316 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-03T05:25:36,318 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-03T05:25:36,318 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-03T05:25:36,321 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-03T05:25:36,321 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-03T05:25:36,325 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733203536322,5,FailOnTimeoutGroup] 2024-12-03T05:25:36,328 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:36,328 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-03T05:25:36,334 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733203536325,5,FailOnTimeoutGroup] 2024-12-03T05:25:36,335 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,335 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-03T05:25:36,337 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,339 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,339 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57437, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T05:25:36,342 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57985, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.1 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T05:25:36,340 INFO [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43493, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.2 (auth:SIMPLE), service=RegionServerStatusService 2024-12-03T05:25:36,366 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,374 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(517): Registering regionserver=e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,430 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,433 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(517): Registering regionserver=e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,434 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 2024-12-03T05:25:36,434 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43427 2024-12-03T05:25:36,435 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T05:25:36,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741831_1007 (size=1321) 2024-12-03T05:25:36,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741831_1007 (size=1321) 2024-12-03T05:25:36,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741831_1007 (size=1321) 2024-12-03T05:25:36,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(363): Checking decommissioned status of RegionServer e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,447 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 2024-12-03T05:25:36,447 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43427 2024-12-03T05:25:36,447 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=37365 {}] master.ServerManager(517): Registering regionserver=e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,447 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T05:25:36,451 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 2024-12-03T05:25:36,451 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:43427 2024-12-03T05:25:36,452 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-03T05:25:36,452 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-03T05:25:36,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:25:36,453 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 2024-12-03T05:25:36,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741832_1008 (size=32) 2024-12-03T05:25:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741832_1008 (size=32) 2024-12-03T05:25:36,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741832_1008 (size=32) 2024-12-03T05:25:36,511 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:36,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T05:25:36,536 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T05:25:36,536 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:36,544 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:36,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T05:25:36,554 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T05:25:36,554 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:36,555 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:36,560 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T05:25:36,567 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T05:25:36,568 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:36,571 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:36,572 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T05:25:36,585 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T05:25:36,585 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:36,592 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:36,593 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T05:25:36,595 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740 2024-12-03T05:25:36,596 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740 2024-12-03T05:25:36,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T05:25:36,613 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T05:25:36,616 DEBUG [RS:1;e3c6570475b6:35935 {}] zookeeper.ZKUtil(111): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,616 DEBUG [RS:0;e3c6570475b6:35421 {}] zookeeper.ZKUtil(111): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,616 WARN [RS:0;e3c6570475b6:35421 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:25:36,616 WARN [RS:1;e3c6570475b6:35935 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:25:36,616 INFO [RS:0;e3c6570475b6:35421 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:36,616 INFO [RS:1;e3c6570475b6:35935 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:36,617 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,617 DEBUG [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,617 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T05:25:36,618 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e3c6570475b6,46191,1733203534664] 2024-12-03T05:25:36,618 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e3c6570475b6,35421,1733203534454] 2024-12-03T05:25:36,618 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e3c6570475b6,35935,1733203534598] 2024-12-03T05:25:36,620 DEBUG [RS:2;e3c6570475b6:46191 {}] zookeeper.ZKUtil(111): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,621 WARN [RS:2;e3c6570475b6:46191 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-03T05:25:36,621 INFO [RS:2;e3c6570475b6:46191 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:36,621 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T05:25:36,621 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,632 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:36,633 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73302732, jitterRate=0.0922958254814148}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T05:25:36,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733203536511Initializing all the Stores at 1733203536521 (+10 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203536521Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203536525 (+4 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203536525Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203536525Cleaning up temporary data from old regions at 1733203536613 (+88 ms)Region opened successfully at 1733203536635 (+22 ms) 2024-12-03T05:25:36,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T05:25:36,636 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T05:25:36,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T05:25:36,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T05:25:36,636 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T05:25:36,640 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T05:25:36,640 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733203536636Disabling compacts and flushes for region at 1733203536636Disabling writes for close at 1733203536636Writing region close event to WAL at 1733203536639 (+3 ms)Closed at 1733203536640 (+1 ms) 2024-12-03T05:25:36,647 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:25:36,647 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-03T05:25:36,656 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-03T05:25:36,673 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T05:25:36,673 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T05:25:36,673 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-03T05:25:36,679 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T05:25:36,688 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-03T05:25:36,702 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T05:25:36,704 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T05:25:36,715 INFO [RS:1;e3c6570475b6:35935 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:25:36,716 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,730 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-03T05:25:36,730 INFO [RS:2;e3c6570475b6:46191 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:25:36,730 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,733 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T05:25:36,735 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T05:25:36,736 INFO [RS:0;e3c6570475b6:35421 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:25:36,736 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,736 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-03T05:25:36,741 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T05:25:36,741 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T05:25:36,741 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-03T05:25:36,743 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,743 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,743 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,743 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,743 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0, corePoolSize=2, maxPoolSize=2 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,744 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,745 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:2;e3c6570475b6:46191 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,745 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,745 DEBUG [RS:0;e3c6570475b6:35421 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,745 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e3c6570475b6:0, corePoolSize=1, maxPoolSize=1 2024-12-03T05:25:36,745 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,745 DEBUG [RS:1;e3c6570475b6:35935 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e3c6570475b6:0, corePoolSize=3, maxPoolSize=3 2024-12-03T05:25:36,760 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,760 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,760 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,760 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,760 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,760 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,35935,1733203534598-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,764 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,765 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,765 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,46191,1733203534664-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:25:36,761 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,765 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,765 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,765 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,35421,1733203534454-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:25:36,791 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T05:25:36,795 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,35935,1733203534598-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,795 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,795 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.Replication(171): e3c6570475b6,35935,1733203534598 started 2024-12-03T05:25:36,795 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T05:25:36,796 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,35421,1733203534454-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,796 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,796 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.Replication(171): e3c6570475b6,35421,1733203534454 started 2024-12-03T05:25:36,801 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-03T05:25:36,802 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,46191,1733203534664-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,802 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,802 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.Replication(171): e3c6570475b6,46191,1733203534664 started 2024-12-03T05:25:36,825 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,825 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1482): Serving as e3c6570475b6,35421,1733203534454, RpcServer on e3c6570475b6/172.17.0.3:35421, sessionid=0x10199a715d70001 2024-12-03T05:25:36,826 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T05:25:36,826 DEBUG [RS:0;e3c6570475b6:35421 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,826 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,35421,1733203534454' 2024-12-03T05:25:36,827 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T05:25:36,828 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T05:25:36,829 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T05:25:36,829 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T05:25:36,829 DEBUG [RS:0;e3c6570475b6:35421 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e3c6570475b6,35421,1733203534454 2024-12-03T05:25:36,829 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,35421,1733203534454' 2024-12-03T05:25:36,829 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T05:25:36,830 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T05:25:36,830 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,830 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1482): Serving as e3c6570475b6,35935,1733203534598, RpcServer on e3c6570475b6/172.17.0.3:35935, sessionid=0x10199a715d70002 2024-12-03T05:25:36,830 DEBUG [RS:0;e3c6570475b6:35421 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T05:25:36,831 INFO [RS:0;e3c6570475b6:35421 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T05:25:36,831 INFO [RS:0;e3c6570475b6:35421 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T05:25:36,831 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T05:25:36,831 DEBUG [RS:1;e3c6570475b6:35935 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,831 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,35935,1733203534598' 2024-12-03T05:25:36,831 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T05:25:36,831 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T05:25:36,831 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:36,832 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1482): Serving as e3c6570475b6,46191,1733203534664, RpcServer on e3c6570475b6/172.17.0.3:46191, sessionid=0x10199a715d70003 2024-12-03T05:25:36,832 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T05:25:36,832 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-03T05:25:36,832 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T05:25:36,832 DEBUG [RS:2;e3c6570475b6:46191 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,832 DEBUG [RS:1;e3c6570475b6:35935 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e3c6570475b6,35935,1733203534598 2024-12-03T05:25:36,832 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,46191,1733203534664' 2024-12-03T05:25:36,832 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,35935,1733203534598' 2024-12-03T05:25:36,832 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-03T05:25:36,832 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T05:25:36,833 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T05:25:36,833 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-03T05:25:36,833 DEBUG [RS:1;e3c6570475b6:35935 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T05:25:36,833 INFO [RS:1;e3c6570475b6:35935 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T05:25:36,833 INFO [RS:1;e3c6570475b6:35935 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T05:25:36,833 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-03T05:25:36,833 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-03T05:25:36,834 DEBUG [RS:2;e3c6570475b6:46191 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e3c6570475b6,46191,1733203534664 2024-12-03T05:25:36,834 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e3c6570475b6,46191,1733203534664' 2024-12-03T05:25:36,834 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-03T05:25:36,834 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-03T05:25:36,835 DEBUG [RS:2;e3c6570475b6:46191 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-03T05:25:36,835 INFO [RS:2;e3c6570475b6:46191 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-03T05:25:36,835 INFO [RS:2;e3c6570475b6:46191 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-03T05:25:36,839 WARN [e3c6570475b6:37365 {}] assignment.AssignmentManager(2451): No servers available; cannot place 1 unassigned regions. 2024-12-03T05:25:36,937 INFO [RS:1;e3c6570475b6:35935 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:36,940 INFO [RS:2;e3c6570475b6:46191 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:36,940 INFO [RS:0;e3c6570475b6:35421 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:36,941 INFO [RS:1;e3c6570475b6:35935 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C35935%2C1733203534598, suffix=, logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35935,1733203534598, archiveDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs, maxLogs=32 2024-12-03T05:25:36,943 INFO [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C35421%2C1733203534454, suffix=, logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454, archiveDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs, maxLogs=32 2024-12-03T05:25:36,945 INFO [RS:2;e3c6570475b6:46191 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C46191%2C1733203534664, suffix=, logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664, archiveDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs, maxLogs=32 2024-12-03T05:25:36,962 DEBUG [RS:1;e3c6570475b6:35935 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35935,1733203534598/e3c6570475b6%2C35935%2C1733203534598.1733203536944, exclude list is [], retry=0 2024-12-03T05:25:36,976 DEBUG [RS:2;e3c6570475b6:46191 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664/e3c6570475b6%2C46191%2C1733203534664.1733203536948, exclude list is [], retry=0 2024-12-03T05:25:36,980 DEBUG [RS:0;e3c6570475b6:35421 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454/e3c6570475b6%2C35421%2C1733203534454.1733203536945, exclude list is [], retry=0 2024-12-03T05:25:36,990 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:36,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:36,992 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:36,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:36,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:36,997 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:37,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:37,003 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:37,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:37,149 INFO [RS:1;e3c6570475b6:35935 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35935,1733203534598/e3c6570475b6%2C35935%2C1733203534598.1733203536944 2024-12-03T05:25:37,153 DEBUG [RS:1;e3c6570475b6:35935 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:37,177 INFO [RS:2;e3c6570475b6:46191 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664/e3c6570475b6%2C46191%2C1733203534664.1733203536948 2024-12-03T05:25:37,180 DEBUG [RS:2;e3c6570475b6:46191 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:37,207 INFO [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454/e3c6570475b6%2C35421%2C1733203534454.1733203536945 2024-12-03T05:25:37,212 DEBUG [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:37,342 DEBUG [e3c6570475b6:37365 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=3, allServersCount=3 2024-12-03T05:25:37,352 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(204): Hosts are {e3c6570475b6=0} racks are {/default-rack=0} 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T05:25:37,363 INFO [e3c6570475b6:37365 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T05:25:37,363 INFO [e3c6570475b6:37365 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T05:25:37,363 INFO [e3c6570475b6:37365 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T05:25:37,363 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T05:25:37,373 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:25:37,380 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3c6570475b6,35421,1733203534454, state=OPENING 2024-12-03T05:25:37,438 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-03T05:25:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:37,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:25:37,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:37,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:37,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:37,450 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:37,452 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-03T05:25:37,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=e3c6570475b6,35421,1733203534454}] 2024-12-03T05:25:37,671 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:25:37,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51965, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:25:37,707 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-03T05:25:37,708 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:37,708 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-03T05:25:37,719 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e3c6570475b6%2C35421%2C1733203534454.meta, suffix=.meta, logDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454, archiveDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs, maxLogs=32 2024-12-03T05:25:37,750 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454/e3c6570475b6%2C35421%2C1733203534454.meta.1733203537722.meta, exclude list is [], retry=0 2024-12-03T05:25:37,755 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:37,756 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:37,757 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:37,831 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,35421,1733203534454/e3c6570475b6%2C35421%2C1733203534454.meta.1733203537722.meta 2024-12-03T05:25:37,832 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:37,836 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:37,838 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-03T05:25:37,841 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-03T05:25:37,852 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-03T05:25:37,857 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-03T05:25:37,858 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:37,858 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-03T05:25:37,858 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-03T05:25:37,876 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-03T05:25:37,885 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-03T05:25:37,885 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:37,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:37,887 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-03T05:25:37,892 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-03T05:25:37,892 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:37,896 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:37,898 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-03T05:25:37,901 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-03T05:25:37,901 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:37,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:37,905 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-03T05:25:37,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-03T05:25:37,913 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:37,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-03T05:25:37,915 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-03T05:25:37,917 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740 2024-12-03T05:25:37,921 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740 2024-12-03T05:25:37,925 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-03T05:25:37,926 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-03T05:25:37,927 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-03T05:25:37,931 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-03T05:25:37,936 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72126587, jitterRate=0.07476989924907684}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-03T05:25:37,936 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-03T05:25:37,938 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733203537859Writing region info on filesystem at 1733203537859Initializing all the Stores at 1733203537864 (+5 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203537864Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203537872 (+8 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203537872Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733203537872Cleaning up temporary data from old regions at 1733203537926 (+54 ms)Running coprocessor post-open hooks at 1733203537936 (+10 ms)Region opened successfully at 1733203537938 (+2 ms) 2024-12-03T05:25:37,957 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733203537659 2024-12-03T05:25:37,980 DEBUG [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-03T05:25:37,984 INFO [RS_OPEN_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-03T05:25:37,990 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:25:38,001 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e3c6570475b6,35421,1733203534454, state=OPEN 2024-12-03T05:25:38,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:25:38,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:25:38,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:25:38,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:38,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:38,061 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:38,062 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=e3c6570475b6,35421,1733203534454 2024-12-03T05:25:38,066 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-03T05:25:38,066 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-03T05:25:38,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-03T05:25:38,089 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=e3c6570475b6,35421,1733203534454 in 608 msec 2024-12-03T05:25:38,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741828_1004 (size=1189) 2024-12-03T05:25:38,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-03T05:25:38,122 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.4330 sec 2024-12-03T05:25:38,134 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-03T05:25:38,135 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-03T05:25:38,173 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:25:38,175 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,35421,1733203534454, seqNum=-1] 2024-12-03T05:25:38,216 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:25:38,223 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44171, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:25:38,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 2.2490 sec 2024-12-03T05:25:38,253 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733203538253, completionTime=-1 2024-12-03T05:25:38,256 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=3; waited=0ms, expected min=3 server(s), max=3 server(s), master is running 2024-12-03T05:25:38,257 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-03T05:25:38,294 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=3 2024-12-03T05:25:38,294 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733203598294 2024-12-03T05:25:38,294 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733203658294 2024-12-03T05:25:38,294 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 37 msec 2024-12-03T05:25:38,296 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(159): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-03T05:25:38,306 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,306 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,306 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,308 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e3c6570475b6:37365, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,309 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,309 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,319 DEBUG [master/e3c6570475b6:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-03T05:25:38,344 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.535sec 2024-12-03T05:25:38,346 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-03T05:25:38,347 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-03T05:25:38,349 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-03T05:25:38,349 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-03T05:25:38,349 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-03T05:25:38,351 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-03T05:25:38,351 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-03T05:25:38,357 DEBUG [master/e3c6570475b6:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-03T05:25:38,358 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-03T05:25:38,359 INFO [master/e3c6570475b6:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e3c6570475b6,37365,1733203533327-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-03T05:25:38,460 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@600d4a87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:25:38,461 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request e3c6570475b6,37365,-1 for getting cluster id 2024-12-03T05:25:38,465 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-03T05:25:38,497 DEBUG [HMaster-EventLoopGroup-2-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'a92647ff-8269-4a17-9036-6ed62d50c959' 2024-12-03T05:25:38,504 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-03T05:25:38,505 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "a92647ff-8269-4a17-9036-6ed62d50c959" 2024-12-03T05:25:38,505 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e250602, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:25:38,505 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [e3c6570475b6,37365,-1] 2024-12-03T05:25:38,509 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-03T05:25:38,514 DEBUG [RPCClient-NioEventLoopGroup-6-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:25:38,529 INFO [HMaster-EventLoopGroup-2-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46918, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-03T05:25:38,541 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6423f813, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-03T05:25:38,541 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-03T05:25:38,566 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=e3c6570475b6,35421,1733203534454, seqNum=-1] 2024-12-03T05:25:38,566 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:25:38,575 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:48830, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:25:38,621 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=e3c6570475b6,37365,1733203533327 2024-12-03T05:25:38,621 INFO [Time-limited test {}] wal.AbstractTestWALReplay(147): hbase.rootdir=hdfs://localhost:43427/hbase 2024-12-03T05:25:38,644 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=363, OpenFileDescriptor=609, MaxFileDescriptor=1048576, SystemLoadAverage=596, ProcessCount=11, AvailableMemoryMB=3900 2024-12-03T05:25:38,698 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:38,712 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:38,717 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:38,736 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-85650159, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-85650159, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:38,757 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-85650159/hregion-85650159.1733203538737, exclude list is [], retry=0 2024-12-03T05:25:38,762 DEBUG [AsyncFSWAL-8-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:38,762 DEBUG [AsyncFSWAL-8-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:38,773 DEBUG [AsyncFSWAL-8-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:38,885 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-85650159/hregion-85650159.1733203538737 2024-12-03T05:25:38,888 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:38,888 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e5830ac447d0f7d29653129d51833365, NAME => 'testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:25:38,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741838_1014 (size=64) 2024-12-03T05:25:38,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741838_1014 (size=64) 2024-12-03T05:25:38,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741838_1014 (size=64) 2024-12-03T05:25:38,964 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:38,972 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,974 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5830ac447d0f7d29653129d51833365 columnFamilyName a 2024-12-03T05:25:38,974 DEBUG [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:38,976 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] regionserver.HStore(327): Store=e5830ac447d0f7d29653129d51833365/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:38,976 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,978 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,979 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,979 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,983 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:38,997 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:38,998 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5830ac447d0f7d29653129d51833365; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70594276, jitterRate=0.05193668603897095}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:25:39,000 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5830ac447d0f7d29653129d51833365: Writing region info on filesystem at 1733203538964Initializing all the Stores at 1733203538971 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203538971Cleaning up temporary data from old regions at 1733203538979 (+8 ms)Region opened successfully at 1733203539000 (+21 ms) 2024-12-03T05:25:39,000 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e5830ac447d0f7d29653129d51833365, disabling compactions & flushes 2024-12-03T05:25:39,000 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. 2024-12-03T05:25:39,000 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. 2024-12-03T05:25:39,000 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. after waiting 0 ms 2024-12-03T05:25:39,000 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. 2024-12-03T05:25:39,001 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. 2024-12-03T05:25:39,001 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e5830ac447d0f7d29653129d51833365: Waiting for close lock at 1733203539000Disabling compacts and flushes for region at 1733203539000Disabling writes for close at 1733203539000Writing region close event to WAL at 1733203539001 (+1 ms)Closed at 1733203539001 2024-12-03T05:25:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741837_1013 (size=175) 2024-12-03T05:25:39,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741837_1013 (size=175) 2024-12-03T05:25:39,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741837_1013 (size=175) 2024-12-03T05:25:39,047 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:39,052 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-85650159:(num 1733203538737) 2024-12-03T05:25:39,060 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T05:25:39,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741839_1015 (size=469) 2024-12-03T05:25:39,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741839_1015 (size=469) 2024-12-03T05:25:39,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741839_1015 (size=469) 2024-12-03T05:25:39,605 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T05:25:39,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741840_1016 (size=322) 2024-12-03T05:25:39,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741840_1016 (size=322) 2024-12-03T05:25:39,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741840_1016 (size=322) 2024-12-03T05:25:40,113 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1, size=469 (469bytes) 2024-12-03T05:25:40,114 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(47): Initialize RecoverLeaseFSUtils 2024-12-03T05:25:40,114 DEBUG [Time-limited test {}] util.RecoverLeaseFSUtils(59): set recoverLeaseMethod to org.apache.hadoop.fs.LeaseRecoverable.recoverLease() 2024-12-03T05:25:40,115 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1 2024-12-03T05:25:40,122 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1 after 4ms 2024-12-03T05:25:40,168 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:40,170 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1 took 59ms 2024-12-03T05:25:40,175 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1 so closing down 2024-12-03T05:25:40,175 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:40,178 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T05:25:40,219 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T05:25:40,221 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:40,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741841_1017 (size=469) 2024-12-03T05:25:40,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741841_1017 (size=469) 2024-12-03T05:25:40,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741841_1017 (size=469) 2024-12-03T05:25:40,251 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:40,259 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002 2024-12-03T05:25:40,264 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-03T05:25:40,264 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1, journal: Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1, size=469 (469bytes) at 1733203540113Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1 so closing down at 1733203540175 (+62 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp at 1733203540219 (+44 ms)3 split writer threads finished at 1733203540221 (+2 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733203540251 (+30 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002 at 1733203540259 (+8 ms)Processed 2 edits across 1 Regions in 88 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733203540264 (+5 ms) 2024-12-03T05:25:40,287 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2, size=322 (322bytes) 2024-12-03T05:25:40,287 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2 2024-12-03T05:25:40,288 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2 after 1ms 2024-12-03T05:25:40,334 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:40,334 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2 took 48ms 2024-12-03T05:25:40,340 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2 so closing down 2024-12-03T05:25:40,340 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:40,347 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T05:25:40,386 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T05:25:40,386 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:40,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741842_1018 (size=322) 2024-12-03T05:25:40,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741842_1018 (size=322) 2024-12-03T05:25:40,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741842_1018 (size=322) 2024-12-03T05:25:40,426 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:40,472 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:40,477 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(184): Found existing old edits file and we have less entries. Deleting hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002-wal-2.temp, length=322 2024-12-03T05:25:40,479 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 143 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-03T05:25:40,479 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2, journal: Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2, size=322 (322bytes) at 1733203540287Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2 so closing down at 1733203540340 (+53 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002-wal-2.temp at 1733203540386 (+46 ms)3 split writer threads finished at 1733203540386Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733203540426 (+40 ms)Processed 1 edits across 1 Regions in 143 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733203540479 (+53 ms) 2024-12-03T05:25:40,480 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:40,483 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:40,498 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal.1733203540484, exclude list is [], retry=0 2024-12-03T05:25:40,502 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:40,503 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:40,504 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:40,552 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal.1733203540484 2024-12-03T05:25:40,553 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:40,553 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e5830ac447d0f7d29653129d51833365, NAME => 'testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:40,553 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:40,554 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,554 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,557 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,559 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e5830ac447d0f7d29653129d51833365 columnFamilyName a 2024-12-03T05:25:40,559 DEBUG [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:40,560 INFO [StoreOpener-e5830ac447d0f7d29653129d51833365-1 {}] regionserver.HStore(327): Store=e5830ac447d0f7d29653129d51833365/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:40,561 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,562 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,565 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,567 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002 2024-12-03T05:25:40,601 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:40,609 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002 2024-12-03T05:25:40,615 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e5830ac447d0f7d29653129d51833365 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T05:25:40,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/.tmp/a/4761094fddc34fec8bba795d37a88993 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733203539053/Put/seqid=0 2024-12-03T05:25:40,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741844_1020 (size=5170) 2024-12-03T05:25:40,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741844_1020 (size=5170) 2024-12-03T05:25:40,687 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741844_1020 (size=5170) 2024-12-03T05:25:40,689 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/.tmp/a/4761094fddc34fec8bba795d37a88993 2024-12-03T05:25:40,737 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/.tmp/a/4761094fddc34fec8bba795d37a88993 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/a/4761094fddc34fec8bba795d37a88993 2024-12-03T05:25:40,764 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/a/4761094fddc34fec8bba795d37a88993, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T05:25:40,814 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e5830ac447d0f7d29653129d51833365 in 157ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T05:25:40,817 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/0000000000000000002 2024-12-03T05:25:40,818 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,818 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,823 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e5830ac447d0f7d29653129d51833365 2024-12-03T05:25:40,829 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/e5830ac447d0f7d29653129d51833365/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T05:25:40,833 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e5830ac447d0f7d29653129d51833365; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70146955, jitterRate=0.04527108371257782}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:25:40,835 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e5830ac447d0f7d29653129d51833365: Writing region info on filesystem at 1733203540554Initializing all the Stores at 1733203540557 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203540557Obtaining lock to block concurrent updates at 1733203540615 (+58 ms)Preparing flush snapshotting stores in e5830ac447d0f7d29653129d51833365 at 1733203540615Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733203540619 (+4 ms)Flushing stores of testReplayEditsWrittenIntoWAL,,1733203538699.e5830ac447d0f7d29653129d51833365. at 1733203540619Flushing e5830ac447d0f7d29653129d51833365/a: creating writer at 1733203540621 (+2 ms)Flushing e5830ac447d0f7d29653129d51833365/a: appending metadata at 1733203540658 (+37 ms)Flushing e5830ac447d0f7d29653129d51833365/a: closing flushed file at 1733203540660 (+2 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3052174c: reopening flushed file at 1733203540735 (+75 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for e5830ac447d0f7d29653129d51833365 in 157ms, sequenceid=2, compaction requested=false; wal=null at 1733203540814 (+79 ms)Cleaning up temporary data from old regions at 1733203540818 (+4 ms)Region opened successfully at 1733203540835 (+17 ms) 2024-12-03T05:25:40,889 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit0 Thread=374 (was 363) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:50694 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:49084 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:49098 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:43427/hbase-prefix:default java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:43612 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: TestAsyncWALReplay-pool-0 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-8-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:43788 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741843_1019, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=700 (was 609) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=588 (was 596), ProcessCount=11 (was 11), AvailableMemoryMB=3634 (was 3900) 2024-12-03T05:25:40,901 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=374, OpenFileDescriptor=699, MaxFileDescriptor=1048576, SystemLoadAverage=588, ProcessCount=11, AvailableMemoryMB=3633 2024-12-03T05:25:40,916 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:40,918 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:40,920 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:40,924 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-36512106, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-36512106, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:40,942 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-36512106/hregion-36512106.1733203540925, exclude list is [], retry=0 2024-12-03T05:25:40,947 DEBUG [AsyncFSWAL-10-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:40,947 DEBUG [AsyncFSWAL-10-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:40,951 DEBUG [AsyncFSWAL-10-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:40,998 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-36512106/hregion-36512106.1733203540925 2024-12-03T05:25:41,000 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:41,001 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => ade59cc4024aba87e68a071db253f7f1, NAME => 'testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:25:41,047 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741846_1022 (size=64) 2024-12-03T05:25:41,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741846_1022 (size=64) 2024-12-03T05:25:41,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741846_1022 (size=64) 2024-12-03T05:25:41,057 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:41,065 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,071 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade59cc4024aba87e68a071db253f7f1 columnFamilyName a 2024-12-03T05:25:41,071 DEBUG [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:41,073 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] regionserver.HStore(327): Store=ade59cc4024aba87e68a071db253f7f1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:41,073 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,075 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,076 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,077 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,077 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,084 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,094 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:41,096 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ade59cc4024aba87e68a071db253f7f1; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70448654, jitterRate=0.049766749143600464}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:25:41,096 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ade59cc4024aba87e68a071db253f7f1: Writing region info on filesystem at 1733203541057Initializing all the Stores at 1733203541060 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203541060Cleaning up temporary data from old regions at 1733203541077 (+17 ms)Region opened successfully at 1733203541096 (+19 ms) 2024-12-03T05:25:41,096 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing ade59cc4024aba87e68a071db253f7f1, disabling compactions & flushes 2024-12-03T05:25:41,097 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. 2024-12-03T05:25:41,097 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. 2024-12-03T05:25:41,097 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. after waiting 0 ms 2024-12-03T05:25:41,097 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. 2024-12-03T05:25:41,100 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. 2024-12-03T05:25:41,101 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for ade59cc4024aba87e68a071db253f7f1: Waiting for close lock at 1733203541096Disabling compacts and flushes for region at 1733203541096Disabling writes for close at 1733203541097 (+1 ms)Writing region close event to WAL at 1733203541100 (+3 ms)Closed at 1733203541100 2024-12-03T05:25:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741845_1021 (size=175) 2024-12-03T05:25:41,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741845_1021 (size=175) 2024-12-03T05:25:41,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741845_1021 (size=175) 2024-12-03T05:25:41,119 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:41,119 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-36512106:(num 1733203540925) 2024-12-03T05:25:41,121 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T05:25:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741847_1023 (size=469) 2024-12-03T05:25:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741847_1023 (size=469) 2024-12-03T05:25:41,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741847_1023 (size=469) 2024-12-03T05:25:41,200 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor testMonitor 2024-12-03T05:25:41,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741848_1024 (size=322) 2024-12-03T05:25:41,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741848_1024 (size=322) 2024-12-03T05:25:41,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741848_1024 (size=322) 2024-12-03T05:25:41,312 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2, size=322 (322bytes) 2024-12-03T05:25:41,312 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2 2024-12-03T05:25:41,313 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2 after 1ms 2024-12-03T05:25:41,367 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:41,367 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2 took 56ms 2024-12-03T05:25:41,372 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2 so closing down 2024-12-03T05:25:41,373 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:41,386 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000002-wal-2.temp 2024-12-03T05:25:41,423 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp 2024-12-03T05:25:41,424 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741849_1025 (size=322) 2024-12-03T05:25:41,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741849_1025 (size=322) 2024-12-03T05:25:41,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741849_1025 (size=322) 2024-12-03T05:25:41,458 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:41,488 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 2024-12-03T05:25:41,489 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 1 edits across 1 Regions in 120 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2, size=322, length=322, corrupted=false, cancelled=false 2024-12-03T05:25:41,489 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2, journal: Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2, size=322 (322bytes) at 1733203541312Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2 so closing down at 1733203541373 (+61 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp at 1733203541423 (+50 ms)3 split writer threads finished at 1733203541424 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733203541458 (+34 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002-wal-2.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 at 1733203541489 (+31 ms)Processed 1 edits across 1 Regions in 120 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-2, size=322, length=322, corrupted=false, cancelled=false at 1733203541489 2024-12-03T05:25:41,511 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1, size=469 (469bytes) 2024-12-03T05:25:41,511 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1 2024-12-03T05:25:41,515 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1 after 3ms 2024-12-03T05:25:41,585 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:41,585 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1 took 75ms 2024-12-03T05:25:41,590 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1 so closing down 2024-12-03T05:25:41,590 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:41,592 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal-1.temp 2024-12-03T05:25:41,625 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp 2024-12-03T05:25:41,626 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:41,651 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741850_1026 (size=469) 2024-12-03T05:25:41,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741850_1026 (size=469) 2024-12-03T05:25:41,652 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741850_1026 (size=469) 2024-12-03T05:25:41,653 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:41,696 DEBUG [split-log-closeStream-pool-0 {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:41,699 WARN [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(175): Found existing old edits file. It could be the result of a previous failed split attempt or we have duplicated wal entries. Deleting hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002, length=322 2024-12-03T05:25:41,703 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 2024-12-03T05:25:41,703 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1, size=469, length=469, corrupted=false, cancelled=false 2024-12-03T05:25:41,703 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1, journal: Splitting hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1, size=469 (469bytes) at 1733203541511Finishing writing output for hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1 so closing down at 1733203541590 (+79 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp at 1733203541625 (+35 ms)3 split writer threads finished at 1733203541626 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733203541653 (+27 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000001-wal-1.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 at 1733203541703 (+50 ms)Processed 2 edits across 1 Regions in 116 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal-1, size=469, length=469, corrupted=false, cancelled=false at 1733203541703 2024-12-03T05:25:41,703 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:41,706 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:41,719 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal.1733203541707, exclude list is [], retry=0 2024-12-03T05:25:41,726 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:41,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:41,727 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:41,772 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal.1733203541707 2024-12-03T05:25:41,773 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:41,773 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ade59cc4024aba87e68a071db253f7f1, NAME => 'testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:41,773 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:41,773 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,774 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,784 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,789 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ade59cc4024aba87e68a071db253f7f1 columnFamilyName a 2024-12-03T05:25:41,790 DEBUG [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:41,791 INFO [StoreOpener-ade59cc4024aba87e68a071db253f7f1-1 {}] regionserver.HStore(327): Store=ade59cc4024aba87e68a071db253f7f1/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:41,792 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,794 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,799 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:41,800 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 2024-12-03T05:25:41,832 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:41,834 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 2, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=2, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 2024-12-03T05:25:41,835 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ade59cc4024aba87e68a071db253f7f1 1/1 column families, dataSize=108 B heapSize=512 B 2024-12-03T05:25:41,867 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/.tmp/a/5457b585be8b495d9da346539a0b4ed4 is 58, key is testReplayEditsWrittenIntoWAL/a:1/1733203541119/Put/seqid=0 2024-12-03T05:25:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741852_1028 (size=5170) 2024-12-03T05:25:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741852_1028 (size=5170) 2024-12-03T05:25:41,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741852_1028 (size=5170) 2024-12-03T05:25:42,298 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=108 B at sequenceid=2 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/.tmp/a/5457b585be8b495d9da346539a0b4ed4 2024-12-03T05:25:42,311 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/.tmp/a/5457b585be8b495d9da346539a0b4ed4 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/a/5457b585be8b495d9da346539a0b4ed4 2024-12-03T05:25:42,322 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/a/5457b585be8b495d9da346539a0b4ed4, entries=2, sequenceid=2, filesize=5.0 K 2024-12-03T05:25:42,323 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ade59cc4024aba87e68a071db253f7f1 in 489ms, sequenceid=2, compaction requested=false; wal=null 2024-12-03T05:25:42,324 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/0000000000000000002 2024-12-03T05:25:42,324 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:42,325 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:42,328 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ade59cc4024aba87e68a071db253f7f1 2024-12-03T05:25:42,333 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/ade59cc4024aba87e68a071db253f7f1/recovered.edits/2.seqid, newMaxSeqId=2, maxSeqId=1 2024-12-03T05:25:42,335 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ade59cc4024aba87e68a071db253f7f1; next sequenceid=3; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70892805, jitterRate=0.056385114789009094}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:25:42,335 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ade59cc4024aba87e68a071db253f7f1: Writing region info on filesystem at 1733203541774Initializing all the Stores at 1733203541781 (+7 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203541781Obtaining lock to block concurrent updates at 1733203541835 (+54 ms)Preparing flush snapshotting stores in ade59cc4024aba87e68a071db253f7f1 at 1733203541835Finished memstore snapshotting testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1., syncing WAL and waiting on mvcc, flushsize=dataSize=108, getHeapSize=496, getOffHeapSize=0, getCellsCount=2 at 1733203541835Flushing stores of testReplayEditsWrittenIntoWAL,,1733203540917.ade59cc4024aba87e68a071db253f7f1. at 1733203541835Flushing ade59cc4024aba87e68a071db253f7f1/a: creating writer at 1733203541835Flushing ade59cc4024aba87e68a071db253f7f1/a: appending metadata at 1733203541865 (+30 ms)Flushing ade59cc4024aba87e68a071db253f7f1/a: closing flushed file at 1733203541865Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c4097c1: reopening flushed file at 1733203542310 (+445 ms)Finished flush of dataSize ~108 B/108, heapSize ~496 B/496, currentSize=0 B/0 for ade59cc4024aba87e68a071db253f7f1 in 489ms, sequenceid=2, compaction requested=false; wal=null at 1733203542323 (+13 ms)Cleaning up temporary data from old regions at 1733203542325 (+2 ms)Region opened successfully at 1733203542335 (+10 ms) 2024-12-03T05:25:42,359 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testNameConflictWhenSplit1 Thread=384 (was 374) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:50768 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-10-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:49098 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:43858 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:43612 [Waiting for operation #24] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:49172 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741851_1027, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=781 (was 699) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=588 (was 588), ProcessCount=11 (was 11), AvailableMemoryMB=3524 (was 3633) 2024-12-03T05:25:42,373 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=384, OpenFileDescriptor=781, MaxFileDescriptor=1048576, SystemLoadAverage=588, ProcessCount=11, AvailableMemoryMB=3524 2024-12-03T05:25:42,396 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:42,399 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:42,401 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:42,405 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64688620, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-64688620, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:42,422 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64688620/hregion-64688620.1733203542406, exclude list is [], retry=0 2024-12-03T05:25:42,425 DEBUG [AsyncFSWAL-12-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:42,425 DEBUG [AsyncFSWAL-12-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:42,438 DEBUG [AsyncFSWAL-12-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:42,476 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64688620/hregion-64688620.1733203542406 2024-12-03T05:25:42,477 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:42,477 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 5b3e51e110dab66885369ee66dcc7b1f, NAME => 'testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenIntoWAL', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:25:42,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741854_1030 (size=64) 2024-12-03T05:25:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741854_1030 (size=64) 2024-12-03T05:25:42,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741854_1030 (size=64) 2024-12-03T05:25:42,664 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-03T05:25:42,911 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:42,914 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,916 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName a 2024-12-03T05:25:42,917 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:42,917 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:42,918 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,920 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName b 2024-12-03T05:25:42,920 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:42,921 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:42,921 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,923 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName c 2024-12-03T05:25:42,923 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:42,924 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:42,928 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,930 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,930 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,932 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,932 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,933 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:42,935 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:42,944 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:42,945 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b3e51e110dab66885369ee66dcc7b1f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62124408, jitterRate=-0.07427418231964111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:42,946 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b3e51e110dab66885369ee66dcc7b1f: Writing region info on filesystem at 1733203542911Initializing all the Stores at 1733203542914 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203542914Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203542914Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203542914Cleaning up temporary data from old regions at 1733203542932 (+18 ms)Region opened successfully at 1733203542945 (+13 ms) 2024-12-03T05:25:42,946 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b3e51e110dab66885369ee66dcc7b1f, disabling compactions & flushes 2024-12-03T05:25:42,946 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:42,946 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:42,946 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. after waiting 0 ms 2024-12-03T05:25:42,946 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:42,947 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:42,947 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b3e51e110dab66885369ee66dcc7b1f: Waiting for close lock at 1733203542946Disabling compacts and flushes for region at 1733203542946Disabling writes for close at 1733203542946Writing region close event to WAL at 1733203542947 (+1 ms)Closed at 1733203542947 2024-12-03T05:25:42,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741853_1029 (size=175) 2024-12-03T05:25:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741853_1029 (size=175) 2024-12-03T05:25:42,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741853_1029 (size=175) 2024-12-03T05:25:42,965 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:42,965 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64688620:(num 1733203542406) 2024-12-03T05:25:42,965 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:42,968 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:42,988 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, exclude list is [], retry=0 2024-12-03T05:25:42,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:42,992 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:42,993 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:43,036 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 2024-12-03T05:25:43,037 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:43,345 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, size=0 (0bytes) 2024-12-03T05:25:43,345 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 might be still open, length is 0 2024-12-03T05:25:43,345 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 2024-12-03T05:25:43,347 WARN [IPC Server handler 2 on default port 43427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 has not been closed. Lease recovery is in progress. RecoveryId = 1032 for block blk_1073741855_1031 2024-12-03T05:25:43,348 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 after 3ms 2024-12-03T05:25:43,370 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:25:43,831 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-03T05:25:43,831 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-03T05:25:43,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T05:25:43,834 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-03T05:25:43,834 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-03T05:25:43,835 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-03T05:25:43,836 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL 2024-12-03T05:25:43,836 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenIntoWAL Metrics about Tables on a single HBase RegionServer 2024-12-03T05:25:44,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38876 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45109:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38876 dst: /127.0.0.1:45109 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45109 remote=/127.0.0.1:38876]. Total timeout mills is 60000, 59129 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:44,158 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:50174 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:40897:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:50174 dst: /127.0.0.1:40897 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:44,158 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38098 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741855_1031] {}] datanode.DataXceiver(331): 127.0.0.1:45403:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:38098 dst: /127.0.0.1:45403 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:44,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741855_1032 (size=497775) 2024-12-03T05:25:44,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741855_1032 (size=497775) 2024-12-03T05:25:44,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741855_1032 (size=497775) 2024-12-03T05:25:47,349 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 after 4004ms 2024-12-03T05:25:47,390 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:47,391 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 took 4046ms 2024-12-03T05:25:47,397 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733203542968.temp 2024-12-03T05:25:47,440 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp 2024-12-03T05:25:47,529 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968; continuing. 2024-12-03T05:25:47,529 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 so closing down 2024-12-03T05:25:47,529 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:47,529 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:47,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741856_1033 (size=497783) 2024-12-03T05:25:47,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741856_1033 (size=497783) 2024-12-03T05:25:47,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741856_1033 (size=497783) 2024-12-03T05:25:47,951 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp (wrote 3002 edits, skipped 0 edits in 44 ms) 2024-12-03T05:25:47,954 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 2024-12-03T05:25:47,954 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3002 edits across 1 Regions in 562 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T05:25:47,954 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, journal: Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, size=0 (0bytes) at 1733203543345Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp at 1733203547440 (+4095 ms)Split 1024 edits, skipped 0 edits. at 1733203547473 (+33 ms)Split 2048 edits, skipped 0 edits. at 1733203547503 (+30 ms)Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 so closing down at 1733203547529 (+26 ms)3 split writer threads finished at 1733203547529Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp (wrote 3002 edits, skipped 0 edits in 44 ms) at 1733203547951 (+422 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000000001-wal.1733203542968.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 at 1733203547954 (+3 ms)Processed 3002 edits across 1 Regions in 562 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968, size=0, length=0, corrupted=false, cancelled=false at 1733203547954 2024-12-03T05:25:47,956 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203542968 2024-12-03T05:25:47,958 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 2024-12-03T05:25:47,958 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:47,961 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:47,976 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203547961, exclude list is [], retry=0 2024-12-03T05:25:47,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:47,980 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:47,981 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:48,016 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203547961 2024-12-03T05:25:48,020 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:48,021 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:48,023 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,025 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName a 2024-12-03T05:25:48,025 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:48,026 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:48,026 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,028 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName b 2024-12-03T05:25:48,028 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:48,029 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:48,029 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,031 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b3e51e110dab66885369ee66dcc7b1f columnFamilyName c 2024-12-03T05:25:48,031 DEBUG [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:48,032 INFO [StoreOpener-5b3e51e110dab66885369ee66dcc7b1f-1 {}] regionserver.HStore(327): Store=5b3e51e110dab66885369ee66dcc7b1f/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:48,032 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,033 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,036 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,037 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 2024-12-03T05:25:48,082 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:48,129 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T05:25:48,522 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b3e51e110dab66885369ee66dcc7b1f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T05:25:48,565 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/b8bc6cb712c749c5a1cdaa2e798b0929 is 62, key is testReplayEditsWrittenIntoWAL/a:100/1733203543047/Put/seqid=0 2024-12-03T05:25:48,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741858_1035 (size=50463) 2024-12-03T05:25:48,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741858_1035 (size=50463) 2024-12-03T05:25:48,587 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=754 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/b8bc6cb712c749c5a1cdaa2e798b0929 2024-12-03T05:25:48,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741858_1035 (size=50463) 2024-12-03T05:25:48,597 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/b8bc6cb712c749c5a1cdaa2e798b0929 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/a/b8bc6cb712c749c5a1cdaa2e798b0929 2024-12-03T05:25:48,605 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/a/b8bc6cb712c749c5a1cdaa2e798b0929, entries=754, sequenceid=754, filesize=49.3 K 2024-12-03T05:25:48,605 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.59 KB/101984, currentSize=0 B/0 for 5b3e51e110dab66885369ee66dcc7b1f in 83ms, sequenceid=754, compaction requested=false; wal=null 2024-12-03T05:25:48,639 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T05:25:48,640 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b3e51e110dab66885369ee66dcc7b1f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T05:25:48,654 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/421c285331334cb6b0cc025993fe101f is 62, key is testReplayEditsWrittenIntoWAL/a:754/1733203543109/Put/seqid=0 2024-12-03T05:25:48,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741859_1036 (size=20072) 2024-12-03T05:25:48,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741859_1036 (size=20072) 2024-12-03T05:25:48,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741859_1036 (size=20072) 2024-12-03T05:25:48,678 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.93 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/421c285331334cb6b0cc025993fe101f 2024-12-03T05:25:48,713 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/29e154c0cd2c4961be47a59c47236415 is 62, key is testReplayEditsWrittenIntoWAL/b:100/1733203543144/Put/seqid=0 2024-12-03T05:25:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741860_1037 (size=35835) 2024-12-03T05:25:48,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741860_1037 (size=35835) 2024-12-03T05:25:48,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741860_1037 (size=35835) 2024-12-03T05:25:48,724 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=28.56 KB at sequenceid=1508 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/29e154c0cd2c4961be47a59c47236415 2024-12-03T05:25:48,739 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/a/421c285331334cb6b0cc025993fe101f as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/a/421c285331334cb6b0cc025993fe101f 2024-12-03T05:25:48,752 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/a/421c285331334cb6b0cc025993fe101f, entries=246, sequenceid=1508, filesize=19.6 K 2024-12-03T05:25:48,754 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/29e154c0cd2c4961be47a59c47236415 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/b/29e154c0cd2c4961be47a59c47236415 2024-12-03T05:25:48,798 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/b/29e154c0cd2c4961be47a59c47236415, entries=508, sequenceid=1508, filesize=35.0 K 2024-12-03T05:25:48,798 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 5b3e51e110dab66885369ee66dcc7b1f in 158ms, sequenceid=1508, compaction requested=false; wal=null 2024-12-03T05:25:48,813 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T05:25:48,813 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b3e51e110dab66885369ee66dcc7b1f 3/3 column families, dataSize=42.49 KB heapSize=100.11 KB 2024-12-03T05:25:48,822 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/c4c05474b9ee41469738fcfd17d3c87d is 62, key is testReplayEditsWrittenIntoWAL/b:508/1733203543170/Put/seqid=0 2024-12-03T05:25:48,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741861_1038 (size=35082) 2024-12-03T05:25:48,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741861_1038 (size=35082) 2024-12-03T05:25:48,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741861_1038 (size=35082) 2024-12-03T05:25:48,843 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=27.87 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/c4c05474b9ee41469738fcfd17d3c87d 2024-12-03T05:25:48,865 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/454b0a64ef2e4212bf3304a38becc3ab is 62, key is testReplayEditsWrittenIntoWAL/c:100/1733203543219/Put/seqid=0 2024-12-03T05:25:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741862_1039 (size=20825) 2024-12-03T05:25:48,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741862_1039 (size=20825) 2024-12-03T05:25:48,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741862_1039 (size=20825) 2024-12-03T05:25:48,890 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=14.63 KB at sequenceid=2262 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/454b0a64ef2e4212bf3304a38becc3ab 2024-12-03T05:25:48,903 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/b/c4c05474b9ee41469738fcfd17d3c87d as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/b/c4c05474b9ee41469738fcfd17d3c87d 2024-12-03T05:25:48,915 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/b/c4c05474b9ee41469738fcfd17d3c87d, entries=492, sequenceid=2262, filesize=34.3 K 2024-12-03T05:25:48,917 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/454b0a64ef2e4212bf3304a38becc3ab as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/c/454b0a64ef2e4212bf3304a38becc3ab 2024-12-03T05:25:48,928 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/c/454b0a64ef2e4212bf3304a38becc3ab, entries=262, sequenceid=2262, filesize=20.3 K 2024-12-03T05:25:48,928 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~42.49 KB/43512, heapSize ~99.83 KB/102224, currentSize=0 B/0 for 5b3e51e110dab66885369ee66dcc7b1f in 115ms, sequenceid=2262, compaction requested=false; wal=null 2024-12-03T05:25:48,940 WARN [Time-limited test {}] regionserver.HRegion(5722): No family for cell testReplayEditsWrittenIntoWAL/another family:testReplayEditsWrittenIntoWAL/1733203543283/Put/vlen=29/seqid=0 in region testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:48,943 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3001, skipped 1, firstSequenceIdInLog=1, maxSequenceIdInLog=3002, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 2024-12-03T05:25:48,943 INFO [Time-limited test {}] wal.AbstractTestWALReplay$4$1(796): InternalFlushCache Invoked 2024-12-03T05:25:48,944 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 5b3e51e110dab66885369ee66dcc7b1f 3/3 column families, dataSize=41.85 KB heapSize=98.89 KB 2024-12-03T05:25:48,963 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/047f3f93aa124fada70a8ea085b184f3 is 62, key is testReplayEditsWrittenIntoWAL/c:262/1733203543229/Put/seqid=0 2024-12-03T05:25:48,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741863_1040 (size=50301) 2024-12-03T05:25:48,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741863_1040 (size=50301) 2024-12-03T05:25:48,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741863_1040 (size=50301) 2024-12-03T05:25:48,978 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=41.85 KB at sequenceid=3002 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/047f3f93aa124fada70a8ea085b184f3 2024-12-03T05:25:48,984 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 047f3f93aa124fada70a8ea085b184f3 2024-12-03T05:25:48,986 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/.tmp/c/047f3f93aa124fada70a8ea085b184f3 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/c/047f3f93aa124fada70a8ea085b184f3 2024-12-03T05:25:48,992 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 047f3f93aa124fada70a8ea085b184f3 2024-12-03T05:25:48,993 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/c/047f3f93aa124fada70a8ea085b184f3, entries=739, sequenceid=3002, filesize=49.1 K 2024-12-03T05:25:48,993 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~41.85 KB/42854, heapSize ~98.38 KB/100736, currentSize=0 B/0 for 5b3e51e110dab66885369ee66dcc7b1f in 49ms, sequenceid=3002, compaction requested=false; wal=null 2024-12-03T05:25:48,994 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/0000000000000003002 2024-12-03T05:25:48,995 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,995 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:48,996 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenIntoWAL descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-12-03T05:25:48,998 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 5b3e51e110dab66885369ee66dcc7b1f 2024-12-03T05:25:49,008 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenIntoWAL/5b3e51e110dab66885369ee66dcc7b1f/recovered.edits/3002.seqid, newMaxSeqId=3002, maxSeqId=1 2024-12-03T05:25:49,009 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 5b3e51e110dab66885369ee66dcc7b1f; next sequenceid=3003; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=204800, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58785806, jitterRate=-0.12402322888374329}}}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-12-03T05:25:49,009 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 5b3e51e110dab66885369ee66dcc7b1f: Writing region info on filesystem at 1733203548021Initializing all the Stores at 1733203548022 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203548022Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203548023 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203548023Cleaning up temporary data from old regions at 1733203548995 (+972 ms)Region opened successfully at 1733203549009 (+14 ms) 2024-12-03T05:25:49,069 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 5b3e51e110dab66885369ee66dcc7b1f, disabling compactions & flushes 2024-12-03T05:25:49,069 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:49,069 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:49,070 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. after waiting 0 ms 2024-12-03T05:25:49,070 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:49,073 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenIntoWAL,,1733203542397.5b3e51e110dab66885369ee66dcc7b1f. 2024-12-03T05:25:49,073 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 5b3e51e110dab66885369ee66dcc7b1f: Waiting for close lock at 1733203549069Disabling compacts and flushes for region at 1733203549069Disabling writes for close at 1733203549070 (+1 ms)Writing region close event to WAL at 1733203549072 (+2 ms)Closed at 1733203549072 2024-12-03T05:25:49,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741857_1034 (size=175) 2024-12-03T05:25:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741857_1034 (size=175) 2024-12-03T05:25:49,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741857_1034 (size=175) 2024-12-03T05:25:49,085 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:49,085 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733203547961) 2024-12-03T05:25:49,109 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenIntoWAL Thread=397 (was 384) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:38479 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261492326_22 at /127.0.0.1:38900 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:38479 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-12-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkins.replay.wal.secondtime@localhost:43427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43427 from jenkins.replay.wal.secondtime java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261492326_22 at /127.0.0.1:50210 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:37539 from jenkins java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-12-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261492326_22 at /127.0.0.1:38128 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Parameter Sending Thread for localhost/127.0.0.1:37539 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1261492326_22 at /127.0.0.1:50246 [Waiting for operation #10] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=859 (was 781) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=589 (was 588) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3111 (was 3524) 2024-12-03T05:25:49,134 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=397, OpenFileDescriptor=861, MaxFileDescriptor=1048576, SystemLoadAverage=589, ProcessCount=11, AvailableMemoryMB=3106 2024-12-03T05:25:49,159 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:49,162 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:49,164 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:49,170 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-03244946, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-03244946, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:49,189 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-03244946/hregion-03244946.1733203549170, exclude list is [], retry=0 2024-12-03T05:25:49,193 DEBUG [AsyncFSWAL-14-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:49,194 DEBUG [AsyncFSWAL-14-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:49,195 DEBUG [AsyncFSWAL-14-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:49,241 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-03244946/hregion-03244946.1733203549170 2024-12-03T05:25:49,256 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:49,257 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e9f1dbc0f6bcb7f133895af5a1ca531b, NAME => 'test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='test2727', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:25:49,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741865_1042 (size=43) 2024-12-03T05:25:49,312 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741865_1042 (size=43) 2024-12-03T05:25:49,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741865_1042 (size=43) 2024-12-03T05:25:49,317 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:49,318 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,320 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName a 2024-12-03T05:25:49,321 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:49,321 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:49,321 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,323 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName b 2024-12-03T05:25:49,323 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:49,325 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:49,326 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,328 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName c 2024-12-03T05:25:49,328 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:49,328 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:49,329 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,330 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,333 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,333 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,334 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:49,341 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:49,352 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:49,353 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f1dbc0f6bcb7f133895af5a1ca531b; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69824937, jitterRate=0.040472641587257385}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f1dbc0f6bcb7f133895af5a1ca531b: Writing region info on filesystem at 1733203549317Initializing all the Stores at 1733203549318 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203549318Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203549318Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203549318Cleaning up temporary data from old regions at 1733203549333 (+15 ms)Region opened successfully at 1733203549355 (+22 ms) 2024-12-03T05:25:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9f1dbc0f6bcb7f133895af5a1ca531b, disabling compactions & flushes 2024-12-03T05:25:49,355 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. after waiting 0 ms 2024-12-03T05:25:49,355 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:49,356 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:49,356 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9f1dbc0f6bcb7f133895af5a1ca531b: Waiting for close lock at 1733203549355Disabling compacts and flushes for region at 1733203549355Disabling writes for close at 1733203549355Writing region close event to WAL at 1733203549356 (+1 ms)Closed at 1733203549356 2024-12-03T05:25:49,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741864_1041 (size=175) 2024-12-03T05:25:49,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741864_1041 (size=175) 2024-12-03T05:25:49,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741864_1041 (size=175) 2024-12-03T05:25:49,378 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:49,379 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-03244946:(num 1733203549170) 2024-12-03T05:25:49,379 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:49,386 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:49,402 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, exclude list is [], retry=0 2024-12-03T05:25:49,406 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:49,407 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:49,407 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:49,452 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 2024-12-03T05:25:49,455 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741866_1043 (size=371388) 2024-12-03T05:25:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741866_1043 (size=371388) 2024-12-03T05:25:49,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741866_1043 (size=371388) 2024-12-03T05:25:49,684 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 not finished, retry = 0 2024-12-03T05:25:49,820 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, size=362.7 K (371388bytes) 2024-12-03T05:25:49,821 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 2024-12-03T05:25:49,825 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 after 4ms 2024-12-03T05:25:49,871 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:49,878 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 took 58ms 2024-12-03T05:25:49,899 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000001-wal.1733203549386.temp 2024-12-03T05:25:49,964 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp 2024-12-03T05:25:49,967 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 so closing down 2024-12-03T05:25:49,967 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:49,987 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:50,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741867_1044 (size=371388) 2024-12-03T05:25:50,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741867_1044 (size=371388) 2024-12-03T05:25:50,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741867_1044 (size=371388) 2024-12-03T05:25:50,428 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp (wrote 3000 edits, skipped 0 edits in 22 ms) 2024-12-03T05:25:50,430 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp to hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 2024-12-03T05:25:50,431 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 547 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, size=362.7 K, length=371388, corrupted=false, cancelled=false 2024-12-03T05:25:50,431 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, journal: Splitting hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, size=362.7 K (371388bytes) at 1733203549820Split 1024 edits, skipped 0 edits. at 1733203549909 (+89 ms)Split 2048 edits, skipped 0 edits. at 1733203549941 (+32 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp at 1733203549964 (+23 ms)Finishing writing output for hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 so closing down at 1733203549967 (+3 ms)3 split writer threads finished at 1733203549987 (+20 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp (wrote 3000 edits, skipped 0 edits in 22 ms) at 1733203550429 (+442 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000000001-wal.1733203549386.temp to hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 at 1733203550430 (+1 ms)Processed 3000 edits across 1 Regions in 547 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386, size=362.7 K, length=371388, corrupted=false, cancelled=false at 1733203550431 (+1 ms) 2024-12-03T05:25:50,433 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203549386 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203549386 2024-12-03T05:25:50,435 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 2024-12-03T05:25:50,435 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:50,438 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:50,457 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, exclude list is [], retry=0 2024-12-03T05:25:50,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:50,461 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:50,462 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:50,512 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 2024-12-03T05:25:50,516 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:50,717 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741868_1045 (size=371515) 2024-12-03T05:25:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741868_1045 (size=371515) 2024-12-03T05:25:50,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741868_1045 (size=371515) 2024-12-03T05:25:50,739 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, size=362.8 K (371515bytes) 2024-12-03T05:25:50,740 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 2024-12-03T05:25:50,742 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 after 2ms 2024-12-03T05:25:50,788 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:50,790 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 took 51ms 2024-12-03T05:25:50,802 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000003001-wal.1733203550438.temp 2024-12-03T05:25:50,859 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp 2024-12-03T05:25:50,861 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 so closing down 2024-12-03T05:25:50,861 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:50,876 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:50,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741869_1046 (size=371515) 2024-12-03T05:25:50,933 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741869_1046 (size=371515) 2024-12-03T05:25:50,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741869_1046 (size=371515) 2024-12-03T05:25:50,942 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp (wrote 3000 edits, skipped 0 edits in 16 ms) 2024-12-03T05:25:50,956 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp to hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 2024-12-03T05:25:50,956 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 3000 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, size=362.8 K, length=371515, corrupted=false, cancelled=false 2024-12-03T05:25:50,956 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, journal: Splitting hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, size=362.8 K (371515bytes) at 1733203550739Split 1024 edits, skipped 0 edits. at 1733203550820 (+81 ms)Split 2048 edits, skipped 0 edits. at 1733203550839 (+19 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp at 1733203550859 (+20 ms)Finishing writing output for hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 so closing down at 1733203550861 (+2 ms)3 split writer threads finished at 1733203550876 (+15 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp (wrote 3000 edits, skipped 0 edits in 16 ms) at 1733203550942 (+66 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003001-wal.1733203550438.temp to hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 at 1733203550956 (+14 ms)Processed 3000 edits across 1 Regions in 160 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438, size=362.8 K, length=371515, corrupted=false, cancelled=false at 1733203550956 2024-12-03T05:25:50,959 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550438 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203550438 2024-12-03T05:25:50,967 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 2024-12-03T05:25:50,968 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:50,971 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/test2727-manual,16010,1733203549158, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:50,998 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550971, exclude list is [], retry=0 2024-12-03T05:25:51,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:51,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:51,005 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:51,058 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/test2727-manual,16010,1733203549158/wal.1733203550971 2024-12-03T05:25:51,064 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:51,064 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e9f1dbc0f6bcb7f133895af5a1ca531b, NAME => 'test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:51,064 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:51,065 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,065 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,068 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,070 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName a 2024-12-03T05:25:51,071 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:51,071 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:51,071 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,072 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName b 2024-12-03T05:25:51,072 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:51,074 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:51,075 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,076 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e9f1dbc0f6bcb7f133895af5a1ca531b columnFamilyName c 2024-12-03T05:25:51,076 DEBUG [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:51,077 INFO [StoreOpener-e9f1dbc0f6bcb7f133895af5a1ca531b-1 {}] regionserver.HStore(327): Store=e9f1dbc0f6bcb7f133895af5a1ca531b/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:51,077 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,078 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,083 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 2 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:51,084 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 2024-12-03T05:25:51,125 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:51,185 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=1, maxSequenceIdInLog=3000, path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 2024-12-03T05:25:51,193 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 2024-12-03T05:25:51,235 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:51,310 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 3000, skipped 0, firstSequenceIdInLog=3001, maxSequenceIdInLog=6000, path=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 2024-12-03T05:25:51,311 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e9f1dbc0f6bcb7f133895af5a1ca531b 3/3 column families, dataSize=215.51 KB heapSize=657 KB 2024-12-03T05:25:51,344 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/a/bf6e26a3389e49e5b8490b3c3d7926ab is 41, key is test2727/a:100/1733203550522/Put/seqid=0 2024-12-03T05:25:51,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741871_1048 (size=84227) 2024-12-03T05:25:51,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741871_1048 (size=84227) 2024-12-03T05:25:51,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741871_1048 (size=84227) 2024-12-03T05:25:51,765 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/a/bf6e26a3389e49e5b8490b3c3d7926ab 2024-12-03T05:25:51,811 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/b/dff331dcd0f9488abde4f72e01e32535 is 41, key is test2727/b:100/1733203550578/Put/seqid=0 2024-12-03T05:25:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741872_1049 (size=84609) 2024-12-03T05:25:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741872_1049 (size=84609) 2024-12-03T05:25:51,864 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741872_1049 (size=84609) 2024-12-03T05:25:51,866 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/b/dff331dcd0f9488abde4f72e01e32535 2024-12-03T05:25:51,905 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/c/47a3e918da6540548026965b7873bab6 is 41, key is test2727/c:100/1733203550642/Put/seqid=0 2024-12-03T05:25:51,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741873_1050 (size=84609) 2024-12-03T05:25:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741873_1050 (size=84609) 2024-12-03T05:25:51,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741873_1050 (size=84609) 2024-12-03T05:25:52,384 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=71.84 KB at sequenceid=6000 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/c/47a3e918da6540548026965b7873bab6 2024-12-03T05:25:52,392 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/a/bf6e26a3389e49e5b8490b3c3d7926ab as hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/a/bf6e26a3389e49e5b8490b3c3d7926ab 2024-12-03T05:25:52,402 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/a/bf6e26a3389e49e5b8490b3c3d7926ab, entries=2000, sequenceid=6000, filesize=82.3 K 2024-12-03T05:25:52,403 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/b/dff331dcd0f9488abde4f72e01e32535 as hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/b/dff331dcd0f9488abde4f72e01e32535 2024-12-03T05:25:52,411 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/b/dff331dcd0f9488abde4f72e01e32535, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T05:25:52,412 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/.tmp/c/47a3e918da6540548026965b7873bab6 as hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/c/47a3e918da6540548026965b7873bab6 2024-12-03T05:25:52,421 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/c/47a3e918da6540548026965b7873bab6, entries=2000, sequenceid=6000, filesize=82.6 K 2024-12-03T05:25:52,421 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e9f1dbc0f6bcb7f133895af5a1ca531b in 1110ms, sequenceid=6000, compaction requested=false; wal=null 2024-12-03T05:25:52,422 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000003000 2024-12-03T05:25:52,423 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/0000000000000006000 2024-12-03T05:25:52,424 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:52,424 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:52,426 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table test2727 descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:52,427 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e9f1dbc0f6bcb7f133895af5a1ca531b 2024-12-03T05:25:52,430 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/test2727/e9f1dbc0f6bcb7f133895af5a1ca531b/recovered.edits/6000.seqid, newMaxSeqId=6000, maxSeqId=1 2024-12-03T05:25:52,431 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e9f1dbc0f6bcb7f133895af5a1ca531b; next sequenceid=6001; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64218362, jitterRate=-0.04307183623313904}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:52,432 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e9f1dbc0f6bcb7f133895af5a1ca531b: Writing region info on filesystem at 1733203551065Initializing all the Stores at 1733203551066 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203551066Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203551068 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203551068Obtaining lock to block concurrent updates at 1733203551311 (+243 ms)Preparing flush snapshotting stores in e9f1dbc0f6bcb7f133895af5a1ca531b at 1733203551311Finished memstore snapshotting test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b., syncing WAL and waiting on mvcc, flushsize=dataSize=220680, getHeapSize=672720, getOffHeapSize=0, getCellsCount=6000 at 1733203551311Flushing stores of test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. at 1733203551311Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/a: creating writer at 1733203551311Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/a: appending metadata at 1733203551342 (+31 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/a: closing flushed file at 1733203551343 (+1 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/b: creating writer at 1733203551772 (+429 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/b: appending metadata at 1733203551809 (+37 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/b: closing flushed file at 1733203551809Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/c: creating writer at 1733203551883 (+74 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/c: appending metadata at 1733203551904 (+21 ms)Flushing e9f1dbc0f6bcb7f133895af5a1ca531b/c: closing flushed file at 1733203551904Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b14af50: reopening flushed file at 1733203552391 (+487 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b7b4bd: reopening flushed file at 1733203552402 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@38a41b3a: reopening flushed file at 1733203552411 (+9 ms)Finished flush of dataSize ~215.51 KB/220680, heapSize ~656.95 KB/672720, currentSize=0 B/0 for e9f1dbc0f6bcb7f133895af5a1ca531b in 1110ms, sequenceid=6000, compaction requested=false; wal=null at 1733203552421 (+10 ms)Cleaning up temporary data from old regions at 1733203552425 (+4 ms)Region opened successfully at 1733203552432 (+7 ms) 2024-12-03T05:25:52,434 DEBUG [Time-limited test {}] wal.AbstractTestWALReplay(320): region.getOpenSeqNum(): 6001, wal3.id: 0 2024-12-03T05:25:52,434 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e9f1dbc0f6bcb7f133895af5a1ca531b, disabling compactions & flushes 2024-12-03T05:25:52,434 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:52,434 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:52,434 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. after waiting 0 ms 2024-12-03T05:25:52,434 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:52,435 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed test2727,,1733203549160.e9f1dbc0f6bcb7f133895af5a1ca531b. 2024-12-03T05:25:52,436 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e9f1dbc0f6bcb7f133895af5a1ca531b: Waiting for close lock at 1733203552434Disabling compacts and flushes for region at 1733203552434Disabling writes for close at 1733203552434Writing region close event to WAL at 1733203552435 (+1 ms)Closed at 1733203552435 2024-12-03T05:25:52,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741870_1047 (size=175) 2024-12-03T05:25:52,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741870_1047 (size=175) 2024-12-03T05:25:52,440 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741870_1047 (size=175) 2024-12-03T05:25:52,443 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:52,443 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733203550971) 2024-12-03T05:25:52,461 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#test2727 Thread=401 (was 397) Potentially hanging thread: AsyncFSWAL-14-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38978 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:50344 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-14-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38224 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=921 (was 861) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 589) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2795 (was 3106) 2024-12-03T05:25:52,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=401, OpenFileDescriptor=921, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=2795 2024-12-03T05:25:52,486 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:52,493 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:52,494 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor wal.1733203552493 2024-12-03T05:25:52,541 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 2024-12-03T05:25:52,550 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new MockWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:25:52,558 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => ca5532bcbc263a1ca66c11fe495e93ed, NAME => 'testSequentialEditLogSeqNum,,1733203552487.ca5532bcbc263a1ca66c11fe495e93ed.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:52,558 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testSequentialEditLogSeqNum,,1733203552487.ca5532bcbc263a1ca66c11fe495e93ed.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:52,558 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,558 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,559 WARN [Time-limited test {}] regionserver.HRegionFileSystem(836): hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed doesn't exist for region: ca5532bcbc263a1ca66c11fe495e93ed on table testSequentialEditLogSeqNum 2024-12-03T05:25:52,560 WARN [Time-limited test {}] regionserver.HRegionFileSystem(854): .regioninfo file not found for region: ca5532bcbc263a1ca66c11fe495e93ed on table testSequentialEditLogSeqNum 2024-12-03T05:25:52,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741875_1052 (size=62) 2024-12-03T05:25:52,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741875_1052 (size=62) 2024-12-03T05:25:52,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741875_1052 (size=62) 2024-12-03T05:25:52,623 INFO [StoreOpener-ca5532bcbc263a1ca66c11fe495e93ed-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,626 INFO [StoreOpener-ca5532bcbc263a1ca66c11fe495e93ed-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca5532bcbc263a1ca66c11fe495e93ed columnFamilyName a 2024-12-03T05:25:52,626 DEBUG [StoreOpener-ca5532bcbc263a1ca66c11fe495e93ed-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:52,627 INFO [StoreOpener-ca5532bcbc263a1ca66c11fe495e93ed-1 {}] regionserver.HStore(327): Store=ca5532bcbc263a1ca66c11fe495e93ed/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:52,627 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,628 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,629 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,629 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,629 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,641 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for ca5532bcbc263a1ca66c11fe495e93ed 2024-12-03T05:25:52,645 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:52,649 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened ca5532bcbc263a1ca66c11fe495e93ed; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67630482, jitterRate=0.007772713899612427}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:25:52,650 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for ca5532bcbc263a1ca66c11fe495e93ed: Writing region info on filesystem at 1733203552558Initializing all the Stores at 1733203552621 (+63 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203552621Cleaning up temporary data from old regions at 1733203552629 (+8 ms)Region opened successfully at 1733203552650 (+21 ms) 2024-12-03T05:25:52,673 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing ca5532bcbc263a1ca66c11fe495e93ed 1/1 column families, dataSize=770 B heapSize=1.73 KB 2024-12-03T05:25:52,698 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/.tmp/a/3043f6157d574311bb89b54aaedbab3e is 81, key is testSequentialEditLogSeqNum/a:x0/1733203552650/Put/seqid=0 2024-12-03T05:25:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741876_1053 (size=5833) 2024-12-03T05:25:52,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741876_1053 (size=5833) 2024-12-03T05:25:52,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741876_1053 (size=5833) 2024-12-03T05:25:52,709 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=770 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/.tmp/a/3043f6157d574311bb89b54aaedbab3e 2024-12-03T05:25:52,716 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/.tmp/a/3043f6157d574311bb89b54aaedbab3e as hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/a/3043f6157d574311bb89b54aaedbab3e 2024-12-03T05:25:52,724 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/a/3043f6157d574311bb89b54aaedbab3e, entries=10, sequenceid=13, filesize=5.7 K 2024-12-03T05:25:52,725 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~770 B/770, heapSize ~1.72 KB/1760, currentSize=0 B/0 for ca5532bcbc263a1ca66c11fe495e93ed in 52ms, sequenceid=13, compaction requested=false 2024-12-03T05:25:52,725 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for ca5532bcbc263a1ca66c11fe495e93ed: 2024-12-03T05:25:52,731 INFO [sync.0 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T05:25:52,731 INFO [sync.1 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T05:25:52,731 INFO [sync.2 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T05:25:52,731 INFO [sync.3 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T05:25:52,731 INFO [sync.4 {}] wal.FSHLog$SyncRunner(477): interrupted 2024-12-03T05:25:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741874_1051 (size=3256) 2024-12-03T05:25:52,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741874_1051 (size=3256) 2024-12-03T05:25:52,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741874_1051 (size=3256) 2024-12-03T05:25:52,754 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493, size=3.2 K (3256bytes) 2024-12-03T05:25:52,754 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 2024-12-03T05:25:52,755 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 after 1ms 2024-12-03T05:25:52,794 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:52,794 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 took 40ms 2024-12-03T05:25:52,797 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 so closing down 2024-12-03T05:25:52,797 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:52,801 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733203552493.temp 2024-12-03T05:25:52,828 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp 2024-12-03T05:25:52,828 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741877_1054 (size=2650) 2024-12-03T05:25:52,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741877_1054 (size=2650) 2024-12-03T05:25:52,843 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741877_1054 (size=2650) 2024-12-03T05:25:52,845 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp (wrote 15 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:52,847 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp to hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000020 2024-12-03T05:25:52,847 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493, size=3.2 K, length=3256, corrupted=false, cancelled=false 2024-12-03T05:25:52,847 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493, journal: Splitting hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493, size=3.2 K (3256bytes) at 1733203552754Finishing writing output for hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493 so closing down at 1733203552797 (+43 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp at 1733203552828 (+31 ms)3 split writer threads finished at 1733203552829 (+1 ms)Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp (wrote 15 edits, skipped 0 edits in 0 ms) at 1733203552845 (+16 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000003-wal.1733203552493.temp to hdfs://localhost:43427/hbase/data/default/testSequentialEditLogSeqNum/ca5532bcbc263a1ca66c11fe495e93ed/recovered.edits/0000000000000000020 at 1733203552847 (+2 ms)Processed 17 edits across 1 Regions in 53 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testsequentialeditlogseqnum-manual,16010,1733203552485/wal.1733203552493, size=3.2 K, length=3256, corrupted=false, cancelled=false at 1733203552847 2024-12-03T05:25:52,863 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testSequentialEditLogSeqNum Thread=406 (was 401) Potentially hanging thread: sync.3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38978 [Waiting for operation #9] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: sync.0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:38224 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: sync.1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.takeSyncRequest(FSHLog.java:426) app//org.apache.hadoop.hbase.regionserver.wal.FSHLog$SyncRunner.run(FSHLog.java:441) - Thread LEAK? -, OpenFileDescriptor=957 (was 921) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=694 (was 694), ProcessCount=11 (was 11), AvailableMemoryMB=2790 (was 2795) 2024-12-03T05:25:52,877 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=406, OpenFileDescriptor=957, MaxFileDescriptor=1048576, SystemLoadAverage=694, ProcessCount=11, AvailableMemoryMB=2788 2024-12-03T05:25:52,898 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:52,906 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:52,941 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:25:52,943 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-45454400, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-45454400, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:52,962 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-45454400/hregion-45454400.1733203552944, exclude list is [], retry=0 2024-12-03T05:25:52,966 DEBUG [AsyncFSWAL-17-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:52,966 DEBUG [AsyncFSWAL-17-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:52,968 DEBUG [AsyncFSWAL-17-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:53,000 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-45454400/hregion-45454400.1733203552944 2024-12-03T05:25:53,001 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:25:53,001 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 025e636a27f69dcc14875c12d4e5b089, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testRegionMadeOfBulkLoadedFilesOnly', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:25:53,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741879_1056 (size=70) 2024-12-03T05:25:53,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741879_1056 (size=70) 2024-12-03T05:25:53,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741879_1056 (size=70) 2024-12-03T05:25:53,434 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:53,436 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,437 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName a 2024-12-03T05:25:53,438 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,438 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,438 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,440 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName b 2024-12-03T05:25:53,440 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,440 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,440 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,442 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName c 2024-12-03T05:25:53,442 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,442 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,443 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,443 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,444 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,445 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,445 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,445 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:53,446 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,449 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:53,450 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 025e636a27f69dcc14875c12d4e5b089; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67657153, jitterRate=0.008170142769813538}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:53,451 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 025e636a27f69dcc14875c12d4e5b089: Writing region info on filesystem at 1733203553434Initializing all the Stores at 1733203553435 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553435Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553436 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553436Cleaning up temporary data from old regions at 1733203553445 (+9 ms)Region opened successfully at 1733203553451 (+6 ms) 2024-12-03T05:25:53,451 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 025e636a27f69dcc14875c12d4e5b089, disabling compactions & flushes 2024-12-03T05:25:53,451 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:53,451 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:53,451 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. after waiting 0 ms 2024-12-03T05:25:53,451 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:53,452 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:53,452 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 025e636a27f69dcc14875c12d4e5b089: Waiting for close lock at 1733203553451Disabling compacts and flushes for region at 1733203553451Disabling writes for close at 1733203553451Writing region close event to WAL at 1733203553452 (+1 ms)Closed at 1733203553452 2024-12-03T05:25:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741878_1055 (size=175) 2024-12-03T05:25:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741878_1055 (size=175) 2024-12-03T05:25:53,455 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741878_1055 (size=175) 2024-12-03T05:25:53,474 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:53,475 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-45454400:(num 1733203552944) 2024-12-03T05:25:53,475 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:53,482 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:53,501 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, exclude list is [], retry=0 2024-12-03T05:25:53,504 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:53,504 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:53,504 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:53,548 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 2024-12-03T05:25:53,549 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:53,549 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 025e636a27f69dcc14875c12d4e5b089, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:53,549 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:53,549 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,549 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,551 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,552 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName a 2024-12-03T05:25:53,552 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,552 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,553 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,553 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName b 2024-12-03T05:25:53,553 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,554 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,554 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,555 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName c 2024-12-03T05:25:53,555 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:53,556 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:53,556 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,556 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,557 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,558 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,558 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,559 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:53,560 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:53,560 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 025e636a27f69dcc14875c12d4e5b089; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64666997, jitterRate=-0.036386653780937195}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:53,561 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 025e636a27f69dcc14875c12d4e5b089: Writing region info on filesystem at 1733203553549Initializing all the Stores at 1733203553550 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553550Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553551 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203553551Cleaning up temporary data from old regions at 1733203553558 (+7 ms)Region opened successfully at 1733203553561 (+3 ms) 2024-12-03T05:25:53,567 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile is 28, key is \x0D/a:a/1733203553565/Put/seqid=0 2024-12-03T05:25:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741881_1058 (size=4826) 2024-12-03T05:25:53,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741881_1058 (size=4826) 2024-12-03T05:25:53,587 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741881_1058 (size=4826) 2024-12-03T05:25:53,589 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile for inclusion in 025e636a27f69dcc14875c12d4e5b089/a 2024-12-03T05:25:53,603 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first= last=z 2024-12-03T05:25:53,603 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T05:25:53,604 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 025e636a27f69dcc14875c12d4e5b089: 2024-12-03T05:25:53,609 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile as hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_ 2024-12-03T05:25:53,610 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 025e636a27f69dcc14875c12d4e5b089/a as hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_ - updating store file list. 2024-12-03T05:25:53,622 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T05:25:53,623 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_ into 025e636a27f69dcc14875c12d4e5b089/a 2024-12-03T05:25:53,623 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43427/hbase/testRegionMadeOfBulkLoadedFilesOnly/hfile into 025e636a27f69dcc14875c12d4e5b089/a (new location: hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_) 2024-12-03T05:25:53,692 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, size=0 (0bytes) 2024-12-03T05:25:53,693 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 might be still open, length is 0 2024-12-03T05:25:53,693 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 2024-12-03T05:25:53,696 WARN [IPC Server handler 1 on default port 43427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 has not been closed. Lease recovery is in progress. RecoveryId = 1059 for block blk_1073741880_1057 2024-12-03T05:25:53,703 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 after 10ms 2024-12-03T05:25:53,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly 2024-12-03T05:25:53,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testRegionMadeOfBulkLoadedFilesOnly Metrics about Tables on a single HBase RegionServer 2024-12-03T05:25:53,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum 2024-12-03T05:25:53,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testSequentialEditLogSeqNum Metrics about Tables on a single HBase RegionServer 2024-12-03T05:25:56,153 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:52160 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45109:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52160 dst: /127.0.0.1:45109 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45109 remote=/127.0.0.1:52160]. Total timeout mills is 60000, 57476 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:56,154 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:46976 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:40897:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:46976 dst: /127.0.0.1:40897 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:56,154 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:52646 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741880_1057] {}] datanode.DataXceiver(331): 127.0.0.1:45403:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:52646 dst: /127.0.0.1:45403 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:25:56,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741880_1059 (size=606) 2024-12-03T05:25:56,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741880_1059 (size=606) 2024-12-03T05:25:56,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741880_1059 (size=606) 2024-12-03T05:25:57,704 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 after 4010ms 2024-12-03T05:25:57,744 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:57,744 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 took 4052ms 2024-12-03T05:25:57,747 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483; continuing. 2024-12-03T05:25:57,747 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 so closing down 2024-12-03T05:25:57,747 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:25:57,749 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000005-wal.1733203553483.temp 2024-12-03T05:25:57,795 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp 2024-12-03T05:25:57,795 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:25:57,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741882_1060 (size=329) 2024-12-03T05:25:57,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741882_1060 (size=329) 2024-12-03T05:25:57,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741882_1060 (size=329) 2024-12-03T05:25:57,830 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp (wrote 1 edits, skipped 0 edits in 0 ms) 2024-12-03T05:25:57,837 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp to hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 2024-12-03T05:25:57,837 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 2 edits across 1 Regions in 92 ms; skipped=1; WAL=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T05:25:57,837 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, journal: Splitting hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, size=0 (0bytes) at 1733203553693Finishing writing output for hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 so closing down at 1733203557747 (+4054 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp at 1733203557795 (+48 ms)3 split writer threads finished at 1733203557795Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp (wrote 1 edits, skipped 0 edits in 0 ms) at 1733203557831 (+36 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005-wal.1733203553483.temp to hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 at 1733203557837 (+6 ms)Processed 2 edits across 1 Regions in 92 ms; skipped=1; WAL=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483, size=0, length=0, corrupted=false, cancelled=false at 1733203557837 2024-12-03T05:25:57,843 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203553483 2024-12-03T05:25:57,847 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 2024-12-03T05:25:57,848 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:25:57,850 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:25:57,869 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203557851, exclude list is [], retry=0 2024-12-03T05:25:57,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:25:57,872 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:25:57,873 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:25:57,902 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203557851 2024-12-03T05:25:57,902 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:25:57,902 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 025e636a27f69dcc14875c12d4e5b089, NAME => 'testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:57,902 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:57,903 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,903 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,924 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,929 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName a 2024-12-03T05:25:57,929 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:57,953 DEBUG [StoreFileOpener-025e636a27f69dcc14875c12d4e5b089-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_: NONE, but ROW specified in column family configuration 2024-12-03T05:25:57,954 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/c219ac0f23e044aa8020eb376cc9efbf_SeqId_3_ 2024-12-03T05:25:57,954 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:57,954 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,955 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName b 2024-12-03T05:25:57,955 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:57,956 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:57,956 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,957 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 025e636a27f69dcc14875c12d4e5b089 columnFamilyName c 2024-12-03T05:25:57,957 DEBUG [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:57,958 INFO [StoreOpener-025e636a27f69dcc14875c12d4e5b089-1 {}] regionserver.HStore(327): Store=025e636a27f69dcc14875c12d4e5b089/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:57,958 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,959 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,961 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:57,962 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 2024-12-03T05:25:58,007 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:25:58,009 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 1, skipped 0, firstSequenceIdInLog=5, maxSequenceIdInLog=5, path=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 2024-12-03T05:25:58,009 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 025e636a27f69dcc14875c12d4e5b089 3/3 column families, dataSize=58 B heapSize=904 B 2024-12-03T05:25:58,037 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/.tmp/a/0c48c3b3b44d4cc983c861dd7fe1b570 is 62, key is testRegionMadeOfBulkLoadedFilesOnly/a:a/1733203553629/Put/seqid=0 2024-12-03T05:25:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741884_1062 (size=5149) 2024-12-03T05:25:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741884_1062 (size=5149) 2024-12-03T05:25:58,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741884_1062 (size=5149) 2024-12-03T05:25:58,056 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58 B at sequenceid=5 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/.tmp/a/0c48c3b3b44d4cc983c861dd7fe1b570 2024-12-03T05:25:58,064 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/.tmp/a/0c48c3b3b44d4cc983c861dd7fe1b570 as hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/0c48c3b3b44d4cc983c861dd7fe1b570 2024-12-03T05:25:58,069 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/a/0c48c3b3b44d4cc983c861dd7fe1b570, entries=1, sequenceid=5, filesize=5.0 K 2024-12-03T05:25:58,069 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 025e636a27f69dcc14875c12d4e5b089 in 60ms, sequenceid=5, compaction requested=false; wal=null 2024-12-03T05:25:58,070 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/0000000000000000005 2024-12-03T05:25:58,071 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:58,071 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:58,072 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testRegionMadeOfBulkLoadedFilesOnly descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:25:58,073 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 025e636a27f69dcc14875c12d4e5b089 2024-12-03T05:25:58,075 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testRegionMadeOfBulkLoadedFilesOnly/025e636a27f69dcc14875c12d4e5b089/recovered.edits/5.seqid, newMaxSeqId=5, maxSeqId=1 2024-12-03T05:25:58,076 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 025e636a27f69dcc14875c12d4e5b089; next sequenceid=6; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74813191, jitterRate=0.11480341851711273}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:25:58,076 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 025e636a27f69dcc14875c12d4e5b089: Writing region info on filesystem at 1733203557903Initializing all the Stores at 1733203557917 (+14 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203557917Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203557924 (+7 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203557924Obtaining lock to block concurrent updates at 1733203558009 (+85 ms)Preparing flush snapshotting stores in 025e636a27f69dcc14875c12d4e5b089 at 1733203558009Finished memstore snapshotting testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089., syncing WAL and waiting on mvcc, flushsize=dataSize=58, getHeapSize=856, getOffHeapSize=0, getCellsCount=1 at 1733203558009Flushing stores of testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. at 1733203558009Flushing 025e636a27f69dcc14875c12d4e5b089/a: creating writer at 1733203558010 (+1 ms)Flushing 025e636a27f69dcc14875c12d4e5b089/a: appending metadata at 1733203558036 (+26 ms)Flushing 025e636a27f69dcc14875c12d4e5b089/a: closing flushed file at 1733203558036Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@57643b64: reopening flushed file at 1733203558063 (+27 ms)Finished flush of dataSize ~58 B/58, heapSize ~376 B/376, currentSize=0 B/0 for 025e636a27f69dcc14875c12d4e5b089 in 60ms, sequenceid=5, compaction requested=false; wal=null at 1733203558069 (+6 ms)Cleaning up temporary data from old regions at 1733203558071 (+2 ms)Region opened successfully at 1733203558076 (+5 ms) 2024-12-03T05:25:58,080 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 025e636a27f69dcc14875c12d4e5b089, disabling compactions & flushes 2024-12-03T05:25:58,080 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:58,081 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:58,081 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. after waiting 0 ms 2024-12-03T05:25:58,081 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:58,081 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testRegionMadeOfBulkLoadedFilesOnly,,1733203552900.025e636a27f69dcc14875c12d4e5b089. 2024-12-03T05:25:58,081 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 025e636a27f69dcc14875c12d4e5b089: Waiting for close lock at 1733203558080Disabling compacts and flushes for region at 1733203558080Disabling writes for close at 1733203558081 (+1 ms)Writing region close event to WAL at 1733203558081Closed at 1733203558081 2024-12-03T05:25:58,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741883_1061 (size=175) 2024-12-03T05:25:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741883_1061 (size=175) 2024-12-03T05:25:58,090 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741883_1061 (size=175) 2024-12-03T05:25:58,093 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:25:58,093 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733203557851) 2024-12-03T05:25:58,112 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testRegionMadeOfBulkLoadedFilesOnly Thread=413 (was 406) Potentially hanging thread: AsyncFSWAL-17-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515271503_22 at /127.0.0.1:47016 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestRegionMadeOfBulkLoadedFilesOnly@localhost:43427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43427 from jenkinstestRegionMadeOfBulkLoadedFilesOnly java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-17-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515271503_22 at /127.0.0.1:52664 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1515271503_22 at /127.0.0.1:52192 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-17-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1023 (was 957) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=671 (was 694), ProcessCount=11 (was 11), AvailableMemoryMB=3575 (was 2788) - AvailableMemoryMB LEAK? - 2024-12-03T05:25:58,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=413, OpenFileDescriptor=1023, MaxFileDescriptor=1048576, SystemLoadAverage=671, ProcessCount=11, AvailableMemoryMB=3575 2024-12-03T05:25:58,148 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:25:58,155 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-03T05:25:58,159 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.AsyncConnectionImpl(321): The fetched master address is e3c6570475b6,37365,1733203533327 2024-12-03T05:25:58,162 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5de84183 2024-12-03T05:25:58,167 DEBUG [RPCClient-NioEventLoopGroup-6-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-03T05:25:58,177 INFO [HMaster-EventLoopGroup-2-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37260, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-03T05:25:58,182 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1'}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-03T05:25:58,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T05:25:58,200 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_PRE_OPERATION 2024-12-03T05:25:58,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "testReplayEditsAfterRegionMovedWithMultiCF" procId is: 4 2024-12-03T05:25:58,203 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:58,207 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-03T05:25:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:25:58,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741885_1063 (size=694) 2024-12-03T05:25:58,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741885_1063 (size=694) 2024-12-03T05:25:58,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741885_1063 (size=694) 2024-12-03T05:25:58,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:25:58,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:25:58,687 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterRegionMovedWithMultiCF', {TABLE_ATTRIBUTES => {REGION_REPLICATION => '1', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298 2024-12-03T05:25:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741886_1064 (size=77) 2024-12-03T05:25:58,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741886_1064 (size=77) 2024-12-03T05:25:58,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741886_1064 (size=77) 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:25:58,722 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:58,722 INFO [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:58,722 DEBUG [RegionOpenAndInit-testReplayEditsAfterRegionMovedWithMultiCF-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203558722Disabling compacts and flushes for region at 1733203558722Disabling writes for close at 1733203558722Writing region close event to WAL at 1733203558722Closed at 1733203558722 2024-12-03T05:25:58,725 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ADD_TO_META 2024-12-03T05:25:58,734 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.","families":{"info":[{"qualifier":"regioninfo","vlen":76,"tag":[],"timestamp":"1733203558725"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733203558725"}]},"ts":"1733203558725"} 2024-12-03T05:25:58,742 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-03T05:25:58,744 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-03T05:25:58,747 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203558744"}]},"ts":"1733203558744"} 2024-12-03T05:25:58,755 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLING in hbase:meta 2024-12-03T05:25:58,756 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(204): Hosts are {e3c6570475b6=0} racks are {/default-rack=0} 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(303): server 2 has 0 regions 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(310): server 2 is on host 0 2024-12-03T05:25:58,766 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T05:25:58,766 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T05:25:58,766 INFO [PEWorker-4 {}] balancer.BalancerClusterState(321): server 2 is on rack 0 2024-12-03T05:25:58,766 DEBUG [PEWorker-4 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T05:25:58,768 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN}] 2024-12-03T05:25:58,772 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN 2024-12-03T05:25:58,776 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN; state=OFFLINE, location=e3c6570475b6,46191,1733203534664; forceNewPlan=false, retain=false 2024-12-03T05:25:58,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:25:58,930 INFO [e3c6570475b6:37365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T05:25:58,931 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPENING, regionLocation=e3c6570475b6,46191,1733203534664 2024-12-03T05:25:58,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN because future has completed 2024-12-03T05:25:58,936 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664}] 2024-12-03T05:25:59,091 DEBUG [RSProcedureDispatcher-pool-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:25:59,093 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35047, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:25:59,099 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,099 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:59,100 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,100 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:59,100 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,100 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,104 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,107 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:25:59,107 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:59,111 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:59,112 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,125 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:25:59,125 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:59,132 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:59,132 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,134 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,140 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,144 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,144 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,146 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T05:25:59,147 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,156 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:25:59,157 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 3452b8a301e7ba272dae5f6efd7fe4dd; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60225913, jitterRate=-0.10256396234035492}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T05:25:59,157 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,158 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Running coprocessor pre-open hook at 1733203559100Writing region info on filesystem at 1733203559100Initializing all the Stores at 1733203559102 (+2 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203559102Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203559104 (+2 ms)Cleaning up temporary data from old regions at 1733203559144 (+40 ms)Running coprocessor post-open hooks at 1733203559157 (+13 ms)Region opened successfully at 1733203559158 (+1 ms) 2024-12-03T05:25:59,160 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., pid=6, masterSystemTime=1733203559090 2024-12-03T05:25:59,171 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,171 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,192 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPEN, openSeqNum=2, regionLocation=e3c6570475b6,46191,1733203534664 2024-12-03T05:25:59,199 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 because future has completed 2024-12-03T05:25:59,236 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-03T05:25:59,237 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 in 280 msec 2024-12-03T05:25:59,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-12-03T05:25:59,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN in 469 msec 2024-12-03T05:25:59,245 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-03T05:25:59,246 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"testReplayEditsAfterRegionMovedWithMultiCF","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733203559245"}]},"ts":"1733203559245"} 2024-12-03T05:25:59,253 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=testReplayEditsAfterRegionMovedWithMultiCF, state=ENABLED in hbase:meta 2024-12-03T05:25:59,255 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF execute state=CREATE_TABLE_POST_OPERATION 2024-12-03T05:25:59,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=testReplayEditsAfterRegionMovedWithMultiCF in 1.0710 sec 2024-12-03T05:25:59,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-03T05:25:59,370 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(3046): Waiting until all regions of table testReplayEditsAfterRegionMovedWithMultiCF get assigned. Timeout = 60000ms 2024-12-03T05:25:59,370 INFO [RPCClient-NioEventLoopGroup-6-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:testReplayEditsAfterRegionMovedWithMultiCF completed 2024-12-03T05:25:59,371 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T05:25:59,378 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3100): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned to meta. Checking AM states. 2024-12-03T05:25:59,379 INFO [Time-limited test {}] hbase.Waiter(181): Waiting up to [60,000] milli-secs(wait.for.ratio=[1]) 2024-12-03T05:25:59,380 INFO [Time-limited test {}] hbase.HBaseTestingUtil(3120): All regions for table testReplayEditsAfterRegionMovedWithMultiCF assigned. 2024-12-03T05:25:59,395 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=2] 2024-12-03T05:25:59,396 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-03T05:25:59,398 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49886, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-03T05:25:59,417 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3452b8a301e7ba272dae5f6efd7fe4dd, source=e3c6570475b6,46191,1733203534664, destination=e3c6570475b6,35421,1733203534454, warming up region on e3c6570475b6,35421,1733203534454 2024-12-03T05:25:59,419 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3452b8a301e7ba272dae5f6efd7fe4dd, source=e3c6570475b6,46191,1733203534664, destination=e3c6570475b6,35421,1733203534454, running balancer 2024-12-03T05:25:59,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE 2024-12-03T05:25:59,421 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE 2024-12-03T05:25:59,424 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=CLOSING, regionLocation=e3c6570475b6,46191,1733203534664 2024-12-03T05:25:59,427 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE because future has completed 2024-12-03T05:25:59,430 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:25:59,430 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664}] 2024-12-03T05:25:59,434 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:25:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:25:59,436 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,439 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:25:59,440 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:59,440 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:59,440 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,442 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:25:59,442 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:25:59,445 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:25:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:25:59,445 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:25:59,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203559445Disabling compacts and flushes for region at 1733203559445Disabling writes for close at 1733203559445Writing region close event to WAL at 1733203559448 (+3 ms)Closed at 1733203559448 2024-12-03T05:25:59,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] procedure.ProcedureSyncWait(219): waitFor pid=7 2024-12-03T05:25:59,593 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(122): Close 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,593 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:25:59,594 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:25:59,594 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,594 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,594 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:25:59,594 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,594 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(2902): Flushing 3452b8a301e7ba272dae5f6efd7fe4dd 2/2 column families, dataSize=31 B heapSize=616 B 2024-12-03T05:25:59,662 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/4d38bfc971914e66969bfee7f68c7210 is 35, key is r1/cf1:q/1733203559399/Put/seqid=0 2024-12-03T05:25:59,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741887_1065 (size=4783) 2024-12-03T05:25:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741887_1065 (size=4783) 2024-12-03T05:25:59,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741887_1065 (size=4783) 2024-12-03T05:25:59,674 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31 B at sequenceid=5 (bloomFilter=false), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/4d38bfc971914e66969bfee7f68c7210 2024-12-03T05:25:59,682 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/4d38bfc971914e66969bfee7f68c7210 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210 2024-12-03T05:25:59,688 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210, entries=1, sequenceid=5, filesize=4.7 K 2024-12-03T05:25:59,689 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3452b8a301e7ba272dae5f6efd7fe4dd in 95ms, sequenceid=5, compaction requested=false 2024-12-03T05:25:59,689 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'testReplayEditsAfterRegionMovedWithMultiCF' 2024-12-03T05:25:59,694 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T05:25:59,696 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:25:59,696 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203559594Running coprocessor pre-close hooks at 1733203559594Disabling compacts and flushes for region at 1733203559594Disabling writes for close at 1733203559594Obtaining lock to block concurrent updates at 1733203559594Preparing flush snapshotting stores in 3452b8a301e7ba272dae5f6efd7fe4dd at 1733203559594Finished memstore snapshotting testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., syncing WAL and waiting on mvcc, flushsize=dataSize=31, getHeapSize=584, getOffHeapSize=0, getCellsCount=1 at 1733203559595 (+1 ms)Flushing stores of testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. at 1733203559599 (+4 ms)Flushing 3452b8a301e7ba272dae5f6efd7fe4dd/cf1: creating writer at 1733203559599Flushing 3452b8a301e7ba272dae5f6efd7fe4dd/cf1: appending metadata at 1733203559662 (+63 ms)Flushing 3452b8a301e7ba272dae5f6efd7fe4dd/cf1: closing flushed file at 1733203559662Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3c48360e: reopening flushed file at 1733203559681 (+19 ms)Finished flush of dataSize ~31 B/31, heapSize ~344 B/344, currentSize=0 B/0 for 3452b8a301e7ba272dae5f6efd7fe4dd in 95ms, sequenceid=5, compaction requested=false at 1733203559689 (+8 ms)Writing region close event to WAL at 1733203559690 (+1 ms)Running coprocessor post-close hooks at 1733203559694 (+4 ms)Closed at 1733203559696 (+2 ms) 2024-12-03T05:25:59,696 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] regionserver.HRegionServer(3302): Adding 3452b8a301e7ba272dae5f6efd7fe4dd move to e3c6570475b6,35421,1733203534454 record at close sequenceid=5 2024-12-03T05:25:59,699 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=8}] handler.UnassignRegionHandler(157): Closed 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:25:59,699 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=CLOSED 2024-12-03T05:25:59,705 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=8, ppid=7, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 because future has completed 2024-12-03T05:25:59,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-12-03T05:25:59,721 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 in 279 msec 2024-12-03T05:25:59,722 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE; state=CLOSED, location=e3c6570475b6,35421,1733203534454; forceNewPlan=false, retain=false 2024-12-03T05:25:59,873 INFO [e3c6570475b6:37365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T05:25:59,873 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPENING, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:25:59,877 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE because future has completed 2024-12-03T05:25:59,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454}] 2024-12-03T05:26:00,040 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,041 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7752): Opening region: {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:00,041 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,041 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:00,041 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7794): checking encryption for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,041 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(7797): checking classloading for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,045 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,048 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:26:00,049 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:00,064 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210 2024-12-03T05:26:00,065 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:00,065 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,066 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:26:00,066 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:00,067 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:00,068 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1038): replaying wal for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,069 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,080 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,081 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1048): stopping wal replay for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,081 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1060): Cleaning up temporary data for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,082 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T05:26:00,084 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1093): writing seq id for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,085 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1114): Opened 3452b8a301e7ba272dae5f6efd7fe4dd; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72952690, jitterRate=0.08707979321479797}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T05:26:00,086 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,086 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegion(1006): Region open journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Running coprocessor pre-open hook at 1733203560041Writing region info on filesystem at 1733203560041Initializing all the Stores at 1733203560045 (+4 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203560045Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203560045Cleaning up temporary data from old regions at 1733203560081 (+36 ms)Running coprocessor post-open hooks at 1733203560086 (+5 ms)Region opened successfully at 1733203560086 2024-12-03T05:26:00,088 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., pid=9, masterSystemTime=1733203560032 2024-12-03T05:26:00,091 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,091 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=9}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,092 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=7 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPEN, openSeqNum=9, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:00,095 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=9, ppid=7, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 because future has completed 2024-12-03T05:26:00,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=9, resume processing ppid=7 2024-12-03T05:26:00,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, ppid=7, state=SUCCESS, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 in 220 msec 2024-12-03T05:26:00,105 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE in 682 msec 2024-12-03T05:26:00,132 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:26:00,134 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:34160, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:26:00,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 139 connection: 172.17.0.3:49886 deadline: 1733203620139, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=35421 startCode=1733203534454. As of locationSeqNum=5. 2024-12-03T05:26:00,171 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=2 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=2, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=35421 startCode=1733203534454. As of locationSeqNum=5. 2024-12-03T05:26:00,172 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=2 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=35421 startCode=1733203534454. As of locationSeqNum=5. 2024-12-03T05:26:00,172 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=2 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=5 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=35421 startCode=1733203534454. As of locationSeqNum=5. 2024-12-03T05:26:00,289 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 3452b8a301e7ba272dae5f6efd7fe4dd 2/2 column families, dataSize=50 B heapSize=720 B 2024-12-03T05:26:00,312 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/29755bd50a47432991c03f151d3ff2e8 is 29, key is r1/cf1:/1733203560279/DeleteFamily/seqid=0 2024-12-03T05:26:00,327 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741888_1066 (size=4906) 2024-12-03T05:26:00,328 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741888_1066 (size=4906) 2024-12-03T05:26:00,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741888_1066 (size=4906) 2024-12-03T05:26:00,331 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,338 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,357 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf2/d603e2b80cdc421884d52cebca0642e9 is 29, key is r1/cf2:/1733203560279/DeleteFamily/seqid=0 2024-12-03T05:26:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741889_1067 (size=4906) 2024-12-03T05:26:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741889_1067 (size=4906) 2024-12-03T05:26:00,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741889_1067 (size=4906) 2024-12-03T05:26:00,366 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=25 B at sequenceid=12 (bloomFilter=false), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf2/d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,373 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,374 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/29755bd50a47432991c03f151d3ff2e8 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,381 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,382 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T05:26:00,383 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf2/d603e2b80cdc421884d52cebca0642e9 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,391 INFO [Time-limited test {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,392 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9, entries=1, sequenceid=12, filesize=4.8 K 2024-12-03T05:26:00,399 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~50 B/50, heapSize ~688 B/688, currentSize=0 B/0 for 3452b8a301e7ba272dae5f6efd7fe4dd in 109ms, sequenceid=12, compaction requested=false 2024-12-03T05:26:00,399 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 3452b8a301e7ba272dae5f6efd7fe4dd: 2024-12-03T05:26:00,402 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-12-03T05:26:00,403 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3452b8a301e7ba272dae5f6efd7fe4dd/cf1 is initiating major compaction (all files) 2024-12-03T05:26:00,403 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:26:00,404 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:26:00,404 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3452b8a301e7ba272dae5f6efd7fe4dd/cf1 in testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,404 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210, hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8] into tmpdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp, totalSize=9.5 K 2024-12-03T05:26:00,406 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 4d38bfc971914e66969bfee7f68c7210, keycount=1, bloomtype=NONE, size=4.7 K, encoding=NONE, compression=NONE, seqNum=5, earliestPutTs=1733203559399 2024-12-03T05:26:00,407 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 29755bd50a47432991c03f151d3ff2e8, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T05:26:00,421 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3452b8a301e7ba272dae5f6efd7fe4dd#cf1#compaction#16 average throughput is 0.00 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:26:00,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741890_1068 (size=4626) 2024-12-03T05:26:00,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741890_1068 (size=4626) 2024-12-03T05:26:00,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741890_1068 (size=4626) 2024-12-03T05:26:00,440 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf1/c24776419397411c9d72d0a34af9e29a as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/c24776419397411c9d72d0a34af9e29a 2024-12-03T05:26:00,457 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 2 (all) file(s) in 3452b8a301e7ba272dae5f6efd7fe4dd/cf1 of 3452b8a301e7ba272dae5f6efd7fe4dd into c24776419397411c9d72d0a34af9e29a(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:26:00,457 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3452b8a301e7ba272dae5f6efd7fe4dd: 2024-12-03T05:26:00,458 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-12-03T05:26:00,458 DEBUG [Time-limited test {}] regionserver.HStore(1541): 3452b8a301e7ba272dae5f6efd7fe4dd/cf2 is initiating major compaction (all files) 2024-12-03T05:26:00,458 INFO [Time-limited test {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-03T05:26:00,458 INFO [Time-limited test {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-03T05:26:00,458 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of 3452b8a301e7ba272dae5f6efd7fe4dd/cf2 in testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,458 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9] into tmpdir=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp, totalSize=4.8 K 2024-12-03T05:26:00,459 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting d603e2b80cdc421884d52cebca0642e9, keycount=1, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=9223372036854775807 2024-12-03T05:26:00,466 INFO [Time-limited test {}] throttle.PressureAwareThroughputController(145): 3452b8a301e7ba272dae5f6efd7fe4dd#cf2#compaction#17 average throughput is NaN MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-12-03T05:26:00,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741891_1069 (size=4592) 2024-12-03T05:26:00,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741891_1069 (size=4592) 2024-12-03T05:26:00,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741891_1069 (size=4592) 2024-12-03T05:26:00,496 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/.tmp/cf2/366f5a163ef54fb892f547d987986a41 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/366f5a163ef54fb892f547d987986a41 2024-12-03T05:26:00,503 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 1 (all) file(s) in 3452b8a301e7ba272dae5f6efd7fe4dd/cf2 of 3452b8a301e7ba272dae5f6efd7fe4dd into 366f5a163ef54fb892f547d987986a41(size=4.5 K), total size for store is 4.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:26:00,503 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for 3452b8a301e7ba272dae5f6efd7fe4dd: 2024-12-03T05:26:00,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster(2410): Client=jenkins//172.17.0.3 move hri=3452b8a301e7ba272dae5f6efd7fe4dd, source=e3c6570475b6,35421,1733203534454, destination=e3c6570475b6,46191,1733203534664, warming up region on e3c6570475b6,46191,1733203534664 2024-12-03T05:26:00,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster(2414): Client=jenkins//172.17.0.3 move hri=3452b8a301e7ba272dae5f6efd7fe4dd, source=e3c6570475b6,35421,1733203534454, destination=e3c6570475b6,46191,1733203534664, running balancer 2024-12-03T05:26:00,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] procedure2.ProcedureExecutor(1139): Stored pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE 2024-12-03T05:26:00,511 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE 2024-12-03T05:26:00,513 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=CLOSING, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:00,514 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.RSRpcServices(2066): Warmup testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(7855): Warmup {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:00,516 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,517 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE because future has completed 2024-12-03T05:26:00,518 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-12-03T05:26:00,518 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454}] 2024-12-03T05:26:00,518 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:26:00,518 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:00,529 INFO [StoreFileOpener-3452b8a301e7ba272dae5f6efd7fe4dd-cf1-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,529 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,538 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210 2024-12-03T05:26:00,545 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/c24776419397411c9d72d0a34af9e29a 2024-12-03T05:26:00,545 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:00,545 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,547 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:26:00,547 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:00,556 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/366f5a163ef54fb892f547d987986a41 2024-12-03T05:26:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] procedure.ProcedureSyncWait(219): waitFor pid=10 2024-12-03T05:26:00,567 INFO [StoreFileOpener-3452b8a301e7ba272dae5f6efd7fe4dd-cf2-1 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,568 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,568 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:26:00,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:26:00,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46191 {}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203560568Disabling compacts and flushes for region at 1733203560568Disabling writes for close at 1733203560568Writing region close event to WAL at 1733203560576 (+8 ms)Closed at 1733203560576 2024-12-03T05:26:00,673 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(122): Close 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,674 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-12-03T05:26:00,674 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:26:00,674 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,674 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,674 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:26:00,674 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,674 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210, hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8] to archive 2024-12-03T05:26:00,678 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:26:00,682 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210 to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/4d38bfc971914e66969bfee7f68c7210 2024-12-03T05:26:00,684 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8 to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/29755bd50a47432991c03f151d3ff2e8 2024-12-03T05:26:00,695 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9] to archive 2024-12-03T05:26:00,697 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:26:00,699 DEBUG [StoreCloser-testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9 to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/archive/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/d603e2b80cdc421884d52cebca0642e9 2024-12-03T05:26:00,704 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/recovered.edits/17.seqid, newMaxSeqId=17, maxSeqId=8 2024-12-03T05:26:00,705 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:00,705 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203560674Running coprocessor pre-close hooks at 1733203560674Disabling compacts and flushes for region at 1733203560674Disabling writes for close at 1733203560674Writing region close event to WAL at 1733203560700 (+26 ms)Running coprocessor post-close hooks at 1733203560705 (+5 ms)Closed at 1733203560705 2024-12-03T05:26:00,705 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] regionserver.HRegionServer(3302): Adding 3452b8a301e7ba272dae5f6efd7fe4dd move to e3c6570475b6,46191,1733203534664 record at close sequenceid=12 2024-12-03T05:26:00,707 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION, pid=11}] handler.UnassignRegionHandler(157): Closed 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:00,708 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=CLOSED 2024-12-03T05:26:00,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=11, ppid=10, state=RUNNABLE, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 because future has completed 2024-12-03T05:26:00,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=11, resume processing ppid=10 2024-12-03T05:26:00,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, ppid=10, state=SUCCESS, hasLock=false; CloseRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 in 194 msec 2024-12-03T05:26:00,715 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE; state=CLOSED, location=e3c6570475b6,46191,1733203534664; forceNewPlan=false, retain=false 2024-12-03T05:26:00,865 INFO [e3c6570475b6:37365 {}] balancer.BaseLoadBalancer(388): Reassigned 1 regions. 1 retained the pre-restart assignment. 2024-12-03T05:26:00,866 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPENING, regionLocation=e3c6570475b6,46191,1733203534664 2024-12-03T05:26:00,869 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=10, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE because future has completed 2024-12-03T05:26:00,869 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664}] 2024-12-03T05:26:01,026 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,026 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7752): Opening region: {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:01,027 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,027 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:01,027 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7794): checking encryption for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,027 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(7797): checking classloading for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,028 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,029 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:26:01,029 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:01,036 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/c24776419397411c9d72d0a34af9e29a 2024-12-03T05:26:01,036 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:01,036 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,037 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:26:01,037 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:01,043 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/366f5a163ef54fb892f547d987986a41 2024-12-03T05:26:01,043 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:01,043 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1038): replaying wal for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,044 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,045 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,046 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1048): stopping wal replay for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,046 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1060): Cleaning up temporary data for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,047 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T05:26:01,048 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1093): writing seq id for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,049 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1114): Opened 3452b8a301e7ba272dae5f6efd7fe4dd; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67558327, jitterRate=0.006697520613670349}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T05:26:01,049 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,050 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegion(1006): Region open journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Running coprocessor pre-open hook at 1733203561027Writing region info on filesystem at 1733203561027Initializing all the Stores at 1733203561028 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203561028Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203561028Cleaning up temporary data from old regions at 1733203561046 (+18 ms)Running coprocessor post-open hooks at 1733203561049 (+3 ms)Region opened successfully at 1733203561050 (+1 ms) 2024-12-03T05:26:01,051 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., pid=12, masterSystemTime=1733203561022 2024-12-03T05:26:01,053 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,053 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=12}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,054 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=10 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPEN, openSeqNum=18, regionLocation=e3c6570475b6,46191,1733203534664 2024-12-03T05:26:01,056 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=12, ppid=10, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 because future has completed 2024-12-03T05:26:01,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=10 2024-12-03T05:26:01,060 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=10, state=SUCCESS, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,46191,1733203534664 in 189 msec 2024-12-03T05:26:01,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, REOPEN/MOVE in 551 msec 2024-12-03T05:26:01,113 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-03T05:26:01,115 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-5-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49888, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-03T05:26:01,116 ERROR [Time-limited test {}] regionserver.HRegionServer(2442): ***** ABORTING region server e3c6570475b6,46191,1733203534664: testing ***** 2024-12-03T05:26:01,116 ERROR [Time-limited test {}] regionserver.HRegionServer(2447): RegionServer abort: loaded coprocessors are: [org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint] 2024-12-03T05:26:01,119 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for java.lang:type=Memory 2024-12-03T05:26:01,120 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=IPC 2024-12-03T05:26:01,125 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Replication 2024-12-03T05:26:01,127 DEBUG [Time-limited test {}] util.JSONBean(135): Listing beans for Hadoop:service=HBase,name=RegionServer,sub=Server 2024-12-03T05:26:01,139 INFO [Time-limited test {}] regionserver.HRegionServer(2451): Dump of metrics as JSON on abort: { "beans": [ { "name": "java.lang:type=Memory", "modelerType": "sun.management.MemoryImpl", "ObjectPendingFinalizationCount": 0, "HeapMemoryUsage": { "committed": 1048576000, "init": 1048576000, "max": 2306867200, "used": 413293456 }, "NonHeapMemoryUsage": { "committed": 172949504, "init": 7667712, "max": -1, "used": 170385240 }, "Verbose": false, "ObjectName": "java.lang:type=Memory" } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=IPC", "modelerType": "RegionServer,sub=IPC", "tag.Context": "regionserver", "tag.Hostname": "e3c6570475b6", "queueSize": 0, "numCallsInGeneralQueue": 0, "numCallsInReplicationQueue": 0, "numCallsInBulkLoadQueue": 0, "numCallsInPriorityQueue": 0, "numCallsInMetaPriorityQueue": 0, "numOpenConnections": 4, "numActiveHandler": 0, "numActiveGeneralHandler": 0, "numActivePriorityHandler": 0, "numActiveReplicationHandler": 0, "numGeneralCallsDropped": 0, "numLifoModeSwitches": 0, "numCallsInWriteQueue": 0, "numActiveBulkLoadHandler": 0, "numCallsInReadQueue": 0, "numCallsInScanQueue": 0, "numActiveWriteHandler": 0, "numActiveReadHandler": 0, "numActiveScanHandler": 0, "nettyDirectMemoryUsage": 67108864, "nettyTotalPendingOutboundBytes": 0, "nettyMaxPendingOutboundBytes": 0, "receivedBytes": 10271, "exceptions.RegionMovedException": 1, "authenticationSuccesses": 0, "authorizationFailures": 0, "exceptions.requestTooBig": 0, "UnwritableTime_num_ops": 0, "UnwritableTime_min": 0, "UnwritableTime_max": 0, "UnwritableTime_mean": 0, "UnwritableTime_25th_percentile": 0, "UnwritableTime_median": 0, "UnwritableTime_75th_percentile": 0, "UnwritableTime_90th_percentile": 0, "UnwritableTime_95th_percentile": 0, "UnwritableTime_98th_percentile": 0, "UnwritableTime_99th_percentile": 0, "UnwritableTime_99.9th_percentile": 0, "exceptions.OutOfOrderScannerNextException": 0, "exceptions.rpcThrottling": 0, "exceptions.otherExceptions": 0, "ProcessCallTime_num_ops": 39, "ProcessCallTime_min": 0, "ProcessCallTime_max": 68, "ProcessCallTime_mean": 5, "ProcessCallTime_25th_percentile": 1, "ProcessCallTime_median": 2, "ProcessCallTime_75th_percentile": 5, "ProcessCallTime_90th_percentile": 7, "ProcessCallTime_95th_percentile": 28, "ProcessCallTime_98th_percentile": 68, "ProcessCallTime_99th_percentile": 68, "ProcessCallTime_99.9th_percentile": 68, "ProcessCallTime_TimeRangeCount_0-1": 9, "ProcessCallTime_TimeRangeCount_1-3": 10, "ProcessCallTime_TimeRangeCount_3-10": 10, "ProcessCallTime_TimeRangeCount_10-30": 1, "ProcessCallTime_TimeRangeCount_30-100": 1, "exceptions.callQueueTooBig": 0, "QueueCallTime_num_ops": 39, "QueueCallTime_min": 0, "QueueCallTime_max": 2, "QueueCallTime_mean": 0, "QueueCallTime_25th_percentile": 0, "QueueCallTime_median": 0, "QueueCallTime_75th_percentile": 0, "QueueCallTime_90th_percentile": 1, "QueueCallTime_95th_percentile": 1, "QueueCallTime_98th_percentile": 2, "QueueCallTime_99th_percentile": 2, "QueueCallTime_99.9th_percentile": 2, "QueueCallTime_TimeRangeCount_0-1": 30, "QueueCallTime_TimeRangeCount_1-3": 1, "authenticationFailures": 0, "exceptions.multiResponseTooLarge": 0, "exceptions.callDropped": 0, "TotalCallTime_num_ops": 39, "TotalCallTime_min": 0, "TotalCallTime_max": 68, "TotalCallTime_mean": 5, "TotalCallTime_25th_percentile": 1, "TotalCallTime_median": 2, "TotalCallTime_75th_percentile": 6, "TotalCallTime_90th_percentile": 7, "TotalCallTime_95th_percentile": 29, "TotalCallTime_98th_percentile": 68, "TotalCallTime_99th_percentile": 68, "TotalCallTime_99.9th_percentile": 68, "TotalCallTime_TimeRangeCount_0-1": 9, "TotalCallTime_TimeRangeCount_1-3": 10, "TotalCallTime_TimeRangeCount_3-10": 10, "TotalCallTime_TimeRangeCount_10-30": 1, "TotalCallTime_TimeRangeCount_30-100": 1, "exceptions.RegionTooBusyException": 0, "exceptions.FailedSanityCheckException": 0, "ResponseSize_num_ops": 39, "ResponseSize_min": 0, "ResponseSize_max": 1138, "ResponseSize_mean": 159, "ResponseSize_25th_percentile": 2, "ResponseSize_median": 2, "ResponseSize_75th_percentile": 74, "ResponseSize_90th_percentile": 453, "ResponseSize_95th_percentile": 795, "ResponseSize_98th_percentile": 1001, "ResponseSize_99th_percentile": 1069, "ResponseSize_99.9th_percentile": 1131, "ResponseSize_SizeRangeCount_0-10": 22, "ResponseSize_SizeRangeCount_10-100": 4, "ResponseSize_SizeRangeCount_100-1000": 5, "exceptions.UnknownScannerException": 0, "exceptions": 1, "maxOutboundBytesExceeded": 0, "authenticationFallbacks": 0, "exceptions.quotaExceeded": 0, "exceptions.callTimedOut": 0, "exceptions.NotServingRegionException": 0, "authorizationSuccesses": 0, "exceptions.ScannerResetException": 0, "RequestSize_num_ops": 39, "RequestSize_min": 31, "RequestSize_max": 390, "RequestSize_mean": 204, "RequestSize_25th_percentile": 119, "RequestSize_median": 179, "RequestSize_75th_percentile": 330, "RequestSize_90th_percentile": 330, "RequestSize_95th_percentile": 365, "RequestSize_98th_percentile": 380, "RequestSize_99th_percentile": 385, "RequestSize_99.9th_percentile": 389, "RequestSize_SizeRangeCount_0-10": 2, "RequestSize_SizeRangeCount_100-1000": 29, "sentBytes": 5772 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Replication", "modelerType": "RegionServer,sub=Replication", "tag.Context": "regionserver", "tag.Hostname": "e3c6570475b6", "source.shippedHFiles": 0, "Source.ageOfLastShippedOp_num_ops": 0, "Source.ageOfLastShippedOp_min": 0, "Source.ageOfLastShippedOp_max": 0, "Source.ageOfLastShippedOp_mean": 0, "Source.ageOfLastShippedOp_25th_percentile": 0, "Source.ageOfLastShippedOp_median": 0, "Source.ageOfLastShippedOp_75th_percentile": 0, "Source.ageOfLastShippedOp_90th_percentile": 0, "Source.ageOfLastShippedOp_95th_percentile": 0, "Source.ageOfLastShippedOp_98th_percentile": 0, "Source.ageOfLastShippedOp_99th_percentile": 0, "Source.ageOfLastShippedOp_99.9th_percentile": 0, "source.uncleanlyClosedLogs": 0, "source.closedLogsWithUnknownFileLength": 0, "source.walReaderEditsBufferUsage": 0, "source.repeatedLogFileBytes": 0, "source.sizeOfHFileRefsQueue": 0, "source.logReadInBytes": 0, "source.completedRecoverQueues": 0, "source.sizeOfLogQueue": 0, "source.restartedLogReading": 0, "source.failedRecoverQueues": 0, "source.ignoredUncleanlyClosedLogContentsInBytes": 0, "Sink.ageOfLastAppliedOp_num_ops": 0, "Sink.ageOfLastAppliedOp_min": 0, "Sink.ageOfLastAppliedOp_max": 0, "Sink.ageOfLastAppliedOp_mean": 0, "Sink.ageOfLastAppliedOp_25th_percentile": 0, "Sink.ageOfLastAppliedOp_median": 0, "Sink.ageOfLastAppliedOp_75th_percentile": 0, "Sink.ageOfLastAppliedOp_90th_percentile": 0, "Sink.ageOfLastAppliedOp_95th_percentile": 0, "Sink.ageOfLastAppliedOp_98th_percentile": 0, "Sink.ageOfLastAppliedOp_99th_percentile": 0, "Sink.ageOfLastAppliedOp_99.9th_percentile": 0, "source.logEditsRead": 0, "source.numInitializing": 0, "source.shippedOps": 0, "sink.appliedHFiles": 0, "source.logEditsFiltered": 0, "source.shippedBytes": 0, "sink.appliedOps": 0, "source.completedLogs": 0, "source.failedBatches": 0, "sink.failedBatches": 0, "source.shippedBatches": 0, "sink.appliedBatches": 0 } ], "beans": [ { "name": "Hadoop:service=HBase,name=RegionServer,sub=Server", "modelerType": "RegionServer,sub=Server", "tag.zookeeperQuorum": "127.0.0.1:53301", "tag.serverName": "e3c6570475b6,35421,1733203534454", "tag.clusterId": "a92647ff-8269-4a17-9036-6ed62d50c959", "tag.Context": "regionserver", "tag.Hostname": "e3c6570475b6", "regionCount": 1, "storeCount": 4, "hlogFileCount": 8, "hlogFileSize": 0, "storeFileCount": 0, "maxStoreFileCount": 0, "memStoreSize": 74, "memStoreHeapSize": 1248, "memStoreOffHeapSize": 0, "storeFileSize": 0, "storeFileSizeGrowthRate": 0.0, "maxStoreFileAge": 0, "minStoreFileAge": 0, "avgStoreFileAge": 0, "numReferenceFiles": 0, "regionServerStartTime": 1733203534454, "averageRegionSize": 74, "storeFileIndexSize": 0, "staticIndexSize": 0, "staticBloomSize": 0, "bloomFilterRequestsCount": 0, "bloomFilterNegativeResultsCount": 0, "bloomFilterEligibleRequestsCount": 0, "mutationsWithoutWALCount": 0, "mutationsWithoutWALSize": 0, "percentFilesLocal": 0.0, "percentFilesLocalSecondaryRegions": 0.0, "totalBytesRead": 320685, "localBytesRead": 320685, "shortCircuitBytesRead": 0, "zeroCopyBytesRead": 0, "splitQueueLength": 0, "compactionQueueLength": 0, "smallCompactionQueueLength": 0, "largeCompactionQueueLength": 0, "flushQueueLength": 0, "blockCacheFreeSize": 922070024, "blockCacheCount": 0, "blockCacheDataBlockCount": 0, "blockCacheSize": 676856, "blockCacheCountHitPercent": 25.0, "blockCacheExpressHitPercent": 0.0, "l1CacheSize": 676856, "l1CacheFreeSize": 922070024, "l1CacheCount": 0, "l1CacheEvictionCount": 0, "l1CacheHitCount": 1, "l1CacheMissCount": 3, "l1CacheHitRatio": 0.25, "l1CacheMissRatio": 0.75, "l2CacheSize": 0, "l2CacheFreeSize": 0, "l2CacheCount": 0, "l2CacheEvictionCount": 0, "l2CacheHitCount": 0, "l2CacheMissCount": 0, "l2CacheHitRatio": 0.0, "l2CacheMissRatio": 0.0, "mobFileCacheCount": 0, "mobFileCacheHitPercent": 0.0, "readRequestRatePerSecond": 0.0, "writeRequestRatePerSecond": 0.0, "ByteBuffAllocatorHeapAllocationBytes": 227435, "ByteBuffAllocatorPoolAllocationBytes": 532480, "ByteBuffAllocatorHeapAllocationRatio": 0.400849396798649, "ByteBuffAllocatorTotalBufferCount": 186, "ByteBuffAllocatorUsedBufferCount": 1, "activeScanners": 0, "totalRequestCount": 28, "totalRowActionRequestCount": 9, "readRequestCount": 7, "cpRequestCount": 0, "filteredReadRequestCount": 0, "writeRequestCount": 2, "rpcGetRequestCount": 2, "rpcFullScanRequestCount": 1, "rpcScanRequestCount": 9, "rpcMultiRequestCount": 0, "rpcMutateRequestCount": 16, "checkMutateFailedCount": 0, "checkMutatePassedCount": 0, "blockCacheHitCount": 1, "blockCacheHitCountPrimary": 1, "blockCacheHitCachingCount": 0, "blockCacheMissCount": 3, "blockCacheMissCountPrimary": 3, "blockCacheMissCachingCount": 1, "blockCacheEvictionCount": 0, "blockCacheEvictionCountPrimary": 0, "blockCacheFailedInsertionCount": 0, "blockCacheDataMissCount": 3, "blockCacheLeafIndexMissCount": 0, "blockCacheBloomChunkMissCount": 0, "blockCacheMetaMissCount": 0, "blockCacheRootIndexMissCount": 0, "blockCacheIntermediateIndexMissCount": 0, "blockCacheFileInfoMissCount": 0, "blockCacheGeneralBloomMetaMissCount": 0, "blockCacheDeleteFamilyBloomMissCount": 0, "blockCacheTrailerMissCount": 0, "blockCacheDataHitCount": 1, "blockCacheLeafIndexHitCount": 0, "blockCacheBloomChunkHitCount": 0, "blockCacheMetaHitCount": 0, "blockCacheRootIndexHitCount": 0, "blockCacheIntermediateIndexHitCount": 0, "blockCacheFileInfoHitCount": 0, "blockCacheGeneralBloomMetaHitCount": 0, "blockCacheDeleteFamilyBloomHitCount": 0, "blockCacheTrailerHitCount": 0, "updatesBlockedTime": 0, "flushedCellsCount": 0, "compactedCellsCount": 0, "majorCompactedCellsCount": 0, "flushedCellsSize": 0, "compactedCellsSize": 0, "majorCompactedCellsSize": 0, "cellsCountCompactedFromMob": 0, "cellsCountCompactedToMob": 0, "cellsSizeCompactedFromMob": 0, "cellsSizeCompactedToMob": 0, "mobFlushCount": 0, "mobFlushedCellsCount": 0, "mobFlushedCellsSize": 0, "mobScanCellsCount": 0, "mobScanCellsSize": 0, "mobFileCacheAccessCount": 0, "mobFileCacheMissCount": 0, "mobFileCacheEvictedCount": 0, "hedgedReads": 0, "hedgedReadWins": 0, "hedgedReadOpsInCurThread": 0, "blockedRequestCount": 0, "CheckAndMutate_num_ops": 0, "CheckAndMutate_min": 0, "CheckAndMutate_max": 0, "CheckAndMutate_mean": 0, "CheckAndMutate_25th_percentile": 0, "CheckAndMutate_median": 0, "CheckAndMutate_75th_percentile": 0, "CheckAndMutate_90th_percentile": 0, "CheckAndMutate_95th_percentile": 0, "CheckAndMutate_98th_percentile": 0, "CheckAndMutate_99th_percentile": 0, "CheckAndMutate_99.9th_percentile": 0, "MajorCompactionTime_num_ops": 2, "MajorCompactionTime_min": 45, "MajorCompactionTime_max": 51, "MajorCompactionTime_mean": 48, "MajorCompactionTime_25th_percentile": 45, "MajorCompactionTime_median": 46, "MajorCompactionTime_75th_percentile": 51, "MajorCompactionTime_90th_percentile": 51, "MajorCompactionTime_95th_percentile": 51, "MajorCompactionTime_98th_percentile": 51, "MajorCompactionTime_99th_percentile": 51, "MajorCompactionTime_99.9th_percentile": 51, "MajorCompactionTime_TimeRangeCount_30-100": 2, "ScanTime_num_ops": 9, "ScanTime_min": 0, "ScanTime_max": 2, "ScanTime_mean": 0, "ScanTime_25th_percentile": 0, "ScanTime_median": 1, "ScanTime_75th_percentile": 1, "ScanTime_90th_percentile": 2, "ScanTime_95th_percentile": 2, "ScanTime_98th_percentile": 2, "ScanTime_99th_percentile": 2, "ScanTime_99.9th_percentile": 2, "ScanTime_TimeRangeCount_0-1": 4, "ScanTime_TimeRangeCount_1-3": 1, "CheckAndMutateBlockBytesScanned_num_ops": 0, "CheckAndMutateBlockBytesScanned_min": 0, "CheckAndMutateBlockBytesScanned_max": 0, "CheckAndMutateBlockBytesScanned_mean": 0, "CheckAndMutateBlockBytesScanned_25th_percentile": 0, "CheckAndMutateBlockBytesScanned_median": 0, "CheckAndMutateBlockBytesScanned_75th_percentile": 0, "CheckAndMutateBlockBytesScanned_90th_percentile": 0, "CheckAndMutateBlockBytesScanned_95th_percentile": 0, "CheckAndMutateBlockBytesScanned_98th_percentile": 0, "CheckAndMutateBlockBytesScanned_99th_percentile": 0, "CheckAndMutateBlockBytesScanned_99.9th_percentile": 0, "Put_num_ops": 15, "Put_min": 1, "Put_max": 6, "Put_mean": 3, "Put_25th_percentile": 2, "Put_median": 2, "Put_75th_percentile": 5, "Put_90th_percentile": 6, "Put_95th_percentile": 6, "Put_98th_percentile": 6, "Put_99th_percentile": 6, "Put_99.9th_percentile": 6, "Put_TimeRangeCount_0-1": 3, "Put_TimeRangeCount_1-3": 5, "Put_TimeRangeCount_3-10": 5, "splitRequestCount": 0, "AppendBlockBytesScanned_num_ops": 0, "AppendBlockBytesScanned_min": 0, "AppendBlockBytesScanned_max": 0, "AppendBlockBytesScanned_mean": 0, "AppendBlockBytesScanned_25th_percentile": 0, "AppendBlockBytesScanned_median": 0, "AppendBlockBytesScanned_75th_percentile": 0, "AppendBlockBytesScanned_90th_percentile": 0, "AppendBlockBytesScanned_95th_percentile": 0, "AppendBlockBytesScanned_98th_percentile": 0, "AppendBlockBytesScanned_99th_percentile": 0, "AppendBlockBytesScanned_99.9th_percentile": 0, "PutBatch_num_ops": 0, "PutBatch_min": 0, "PutBatch_max": 0, "PutBatch_mean": 0, "PutBatch_25th_percentile": 0, "PutBatch_median": 0, "PutBatch_75th_percentile": 0, "PutBatch_90th_percentile": 0, "PutBatch_95th_percentile": 0, "PutBatch_98th_percentile": 0, "PutBatch_99th_percentile": 0, "PutBatch_99.9th_percentile": 0, "IncrementBlockBytesScanned_num_ops": 0, "IncrementBlockBytesScanned_min": 0, "IncrementBlockBytesScanned_max": 0, "IncrementBlockBytesScanned_mean": 0, "IncrementBlockBytesScanned_25th_percentile": 0, "IncrementBlockBytesScanned_median": 0, "IncrementBlockBytesScanned_75th_percentile": 0, "IncrementBlockBytesScanned_90th_percentile": 0, "IncrementBlockBytesScanned_95th_percentile": 0, "IncrementBlockBytesScanned_98th_percentile": 0, "IncrementBlockBytesScanned_99th_percentile": 0, "IncrementBlockBytesScanned_99.9th_percentile": 0, "SplitTime_num_ops": 0, "SplitTime_min": 0, "SplitTime_max": 0, "SplitTime_mean": 0, "SplitTime_25th_percentile": 0, "SplitTime_median": 0, "SplitTime_75th_percentile": 0, "SplitTime_90th_percentile": 0, "SplitTime_95th_percentile": 0, "SplitTime_98th_percentile": 0, "SplitTime_99th_percentile": 0, "SplitTime_99.9th_percentile": 0, "GetBlockBytesScanned_num_ops": 0, "GetBlockBytesScanned_min": 0, "GetBlockBytesScanned_max": 0, "GetBlockBytesScanned_mean": 0, "GetBlockBytesScanned_25th_percentile": 0, "GetBlockBytesScanned_median": 0, "GetBlockBytesScanned_75th_percentile": 0, "GetBlockBytesScanned_90th_percentile": 0, "GetBlockBytesScanned_95th_percentile": 0, "GetBlockBytesScanned_98th_percentile": 0, "GetBlockBytesScanned_99th_percentile": 0, "GetBlockBytesScanned_99.9th_percentile": 0, "majorCompactedInputBytes": 14595, "slowAppendCount": 0, "flushedOutputBytes": 9812, "Replay_num_ops": 0, "Replay_min": 0, "Replay_max": 0, "Replay_mean": 0, "Replay_25th_percentile": 0, "Replay_median": 0, "Replay_75th_percentile": 0, "Replay_90th_percentile": 0, "Replay_95th_percentile": 0, "Replay_98th_percentile": 0, "Replay_99th_percentile": 0, "Replay_99.9th_percentile": 0, "MajorCompactionInputSize_num_ops": 2, "MajorCompactionInputSize_min": 4906, "MajorCompactionInputSize_max": 9689, "MajorCompactionInputSize_mean": 7297, "MajorCompactionInputSize_25th_percentile": 6101, "MajorCompactionInputSize_median": 7297, "MajorCompactionInputSize_75th_percentile": 8493, "MajorCompactionInputSize_90th_percentile": 9210, "MajorCompactionInputSize_95th_percentile": 9449, "MajorCompactionInputSize_98th_percentile": 9593, "MajorCompactionInputSize_99th_percentile": 9641, "MajorCompactionInputSize_99.9th_percentile": 9684, "MajorCompactionInputSize_SizeRangeCount_100-1000": 2, "pauseInfoThresholdExceeded": 0, "CheckAndDelete_num_ops": 0, "CheckAndDelete_min": 0, "CheckAndDelete_max": 0, "CheckAndDelete_mean": 0, "CheckAndDelete_25th_percentile": 0, "CheckAndDelete_median": 0, "CheckAndDelete_75th_percentile": 0, "CheckAndDelete_90th_percentile": 0, "CheckAndDelete_95th_percentile": 0, "CheckAndDelete_98th_percentile": 0, "CheckAndDelete_99th_percentile": 0, "CheckAndDelete_99.9th_percentile": 0, "CompactionInputSize_num_ops": 2, "CompactionInputSize_min": 4906, "CompactionInputSize_max": 9689, "CompactionInputSize_mean": 7297, "CompactionInputSize_25th_percentile": 6101, "CompactionInputSize_median": 7297, "CompactionInputSize_75th_percentile": 8493, "CompactionInputSize_90th_percentile": 9210, "CompactionInputSize_95th_percentile": 9449, "CompactionInputSize_98th_percentile": 9593, "CompactionInputSize_99th_percentile": 9641, "CompactionInputSize_99.9th_percentile": 9684, "CompactionInputSize_SizeRangeCount_100-1000": 2, "flushedMemstoreBytes": 50, "majorCompactedOutputBytes": 9218, "slowPutCount": 0, "compactedInputBytes": 14595, "FlushOutputSize_num_ops": 1, "FlushOutputSize_min": 9812, "FlushOutputSize_max": 9812, "FlushOutputSize_mean": 9812, "FlushOutputSize_25th_percentile": 9812, "FlushOutputSize_median": 9812, "FlushOutputSize_75th_percentile": 9812, "FlushOutputSize_90th_percentile": 9812, "FlushOutputSize_95th_percentile": 9812, "FlushOutputSize_98th_percentile": 9812, "FlushOutputSize_99th_percentile": 9812, "FlushOutputSize_99.9th_percentile": 9812, "FlushOutputSize_SizeRangeCount_100-1000": 1, "PauseTimeWithGc_num_ops": 0, "PauseTimeWithGc_min": 0, "PauseTimeWithGc_max": 0, "PauseTimeWithGc_mean": 0, "PauseTimeWithGc_25th_percentile": 0, "PauseTimeWithGc_median": 0, "PauseTimeWithGc_75th_percentile": 0, "PauseTimeWithGc_90th_percentile": 0, "PauseTimeWithGc_95th_percentile": 0, "PauseTimeWithGc_98th_percentile": 0, "PauseTimeWithGc_99th_percentile": 0, "PauseTimeWithGc_99.9th_percentile": 0, "compactedOutputBytes": 9218, "pauseWarnThresholdExceeded": 0, "ScanBlockBytesScanned_num_ops": 1, "ScanBlockBytesScanned_min": 32, "ScanBlockBytesScanned_max": 32, "ScanBlockBytesScanned_mean": 32, "ScanBlockBytesScanned_25th_percentile": 32, "ScanBlockBytesScanned_median": 32, "ScanBlockBytesScanned_75th_percentile": 32, "ScanBlockBytesScanned_90th_percentile": 32, "ScanBlockBytesScanned_95th_percentile": 32, "ScanBlockBytesScanned_98th_percentile": 32, "ScanBlockBytesScanned_99th_percentile": 32, "ScanBlockBytesScanned_99.9th_percentile": 32, "ScanBlockBytesScanned_SizeRangeCount_10-100": 1, "Increment_num_ops": 0, "Increment_min": 0, "Increment_max": 0, "Increment_mean": 0, "Increment_25th_percentile": 0, "Increment_median": 0, "Increment_75th_percentile": 0, "Increment_90th_percentile": 0, "Increment_95th_percentile": 0, "Increment_98th_percentile": 0, "Increment_99th_percentile": 0, "Increment_99.9th_percentile": 0, "Delete_num_ops": 1, "Delete_min": 4, "Delete_max": 4, "Delete_mean": 4, "Delete_25th_percentile": 4, "Delete_median": 4, "Delete_75th_percentile": 4, "Delete_90th_percentile": 4, "Delete_95th_percentile": 4, "Delete_98th_percentile": 4, "Delete_99th_percentile": 4, "Delete_99.9th_percentile": 4, "Delete_TimeRangeCount_3-10": 1, "DeleteBatch_num_ops": 0, "DeleteBatch_min": 0, "DeleteBatch_max": 0, "DeleteBatch_mean": 0, "DeleteBatch_25th_percentile": 0, "DeleteBatch_median": 0, "DeleteBatch_75th_percentile": 0, "DeleteBatch_90th_percentile": 0, "DeleteBatch_95th_percentile": 0, "DeleteBatch_98th_percentile": 0, "DeleteBatch_99th_percentile": 0, "DeleteBatch_99.9th_percentile": 0, "blockBytesScannedCount": 32, "FlushMemstoreSize_num_ops": 1, "FlushMemstoreSize_min": 50, "FlushMemstoreSize_max": 50, "FlushMemstoreSize_mean": 50, "FlushMemstoreSize_25th_percentile": 50, "FlushMemstoreSize_median": 50, "FlushMemstoreSize_75th_percentile": 50, "FlushMemstoreSize_90th_percentile": 50, "FlushMemstoreSize_95th_percentile": 50, "FlushMemstoreSize_98th_percentile": 50, "FlushMemstoreSize_99th_percentile": 50, "FlushMemstoreSize_99.9th_percentile": 50, "FlushMemstoreSize_SizeRangeCount_10-100": 1, "CompactionInputFileCount_num_ops": 2, "CompactionInputFileCount_min": 1, "CompactionInputFileCount_max": 2, "CompactionInputFileCount_mean": 1, "CompactionInputFileCount_25th_percentile": 1, "CompactionInputFileCount_median": 2, "CompactionInputFileCount_75th_percentile": 2, "CompactionInputFileCount_90th_percentile": 2, "CompactionInputFileCount_95th_percentile": 2, "CompactionInputFileCount_98th_percentile": 2, "CompactionInputFileCount_99th_percentile": 2, "CompactionInputFileCount_99.9th_percentile": 2, "CompactionTime_num_ops": 2, "CompactionTime_min": 45, "CompactionTime_max": 51, "CompactionTime_mean": 48, "CompactionTime_25th_percentile": 45, "CompactionTime_median": 46, "CompactionTime_75th_percentile": 51, "CompactionTime_90th_percentile": 51, "CompactionTime_95th_percentile": 51, "CompactionTime_98th_percentile": 51, "CompactionTime_99th_percentile": 51, "CompactionTime_99.9th_percentile": 51, "CompactionTime_TimeRangeCount_30-100": 2, "Get_num_ops": 2, "Get_min": 0, "Get_max": 0, "Get_mean": 0, "Get_25th_percentile": 0, "Get_median": 0, "Get_75th_percentile": 0, "Get_90th_percentile": 0, "Get_95th_percentile": 0, "Get_98th_percentile": 0, "Get_99th_percentile": 0, "Get_99.9th_percentile": 0, "Get_TimeRangeCount_0-1": 1, "MajorCompactionInputFileCount_num_ops": 2, "MajorCompactionInputFileCount_min": 1, "MajorCompactionInputFileCount_max": 2, "MajorCompactionInputFileCount_mean": 1, "MajorCompactionInputFileCount_25th_percentile": 1, "MajorCompactionInputFileCount_median": 2, "MajorCompactionInputFileCount_75th_percentile": 2, "MajorCompactionInputFileCount_90th_percentile": 2, "MajorCompactionInputFileCount_95th_percentile": 2, "MajorCompactionInputFileCount_98th_percentile": 2, "MajorCompactionInputFileCount_99th_percentile": 2, "MajorCompactionInputFileCount_99.9th_percentile": 2, "scannerLeaseExpiredCount": 0, "CheckAndPut_num_ops": 0, "CheckAndPut_min": 0, "CheckAndPut_max": 0, "CheckAndPut_mean": 0, "CheckAndPut_25th_percentile": 0, "CheckAndPut_median": 0, "CheckAndPut_75th_percentile": 0, "CheckAndPut_90th_percentile": 0, "CheckAndPut_95th_percentile": 0, "CheckAndPut_98th_percentile": 0, "CheckAndPut_99th_percentile": 0, "CheckAndPut_99.9th_percentile": 0, "MajorCompactionOutputSize_num_ops": 2, "MajorCompactionOutputSize_min": 4592, "MajorCompactionOutputSize_max": 4626, "MajorCompactionOutputSize_mean": 4609, "MajorCompactionOutputSize_25th_percentile": 4600, "MajorCompactionOutputSize_median": 4609, "MajorCompactionOutputSize_75th_percentile": 4617, "MajorCompactionOutputSize_90th_percentile": 4622, "MajorCompactionOutputSize_95th_percentile": 4624, "MajorCompactionOutputSize_98th_percentile": 4625, "MajorCompactionOutputSize_99th_percentile": 4625, "MajorCompactionOutputSize_99.9th_percentile": 4625, "MajorCompactionOutputSize_SizeRangeCount_100-1000": 2, "CompactionOutputFileCount_num_ops": 2, "CompactionOutputFileCount_min": 1, "CompactionOutputFileCount_max": 1, "CompactionOutputFileCount_mean": 1, "CompactionOutputFileCount_25th_percentile": 1, "CompactionOutputFileCount_median": 1, "CompactionOutputFileCount_75th_percentile": 1, "CompactionOutputFileCount_90th_percentile": 1, "CompactionOutputFileCount_95th_percentile": 1, "CompactionOutputFileCount_98th_percentile": 1, "CompactionOutputFileCount_99th_percentile": 1, "CompactionOutputFileCount_99.9th_percentile": 1, "slowDeleteCount": 0, "FlushTime_num_ops": 1, "FlushTime_min": 109, "FlushTime_max": 109, "FlushTime_mean": 109, "FlushTime_25th_percentile": 109, "FlushTime_median": 109, "FlushTime_75th_percentile": 109, "FlushTime_90th_percentile": 109, "FlushTime_95th_percentile": 109, "FlushTime_98th_percentile": 109, "FlushTime_99th_percentile": 109, "FlushTime_99.9th_percentile": 109, "FlushTime_TimeRangeCount_100-300": 1, "splitSuccessCount": 0, "MajorCompactionOutputFileCount_num_ops": 2, "MajorCompactionOutputFileCount_min": 1, "MajorCompactionOutputFileCount_max": 1, "MajorCompactionOutputFileCount_mean": 1, "MajorCompactionOutputFileCount_25th_percentile": 1, "MajorCompactionOutputFileCount_median": 1, "MajorCompactionOutputFileCount_75th_percentile": 1, "MajorCompactionOutputFileCount_90th_percentile": 1, "MajorCompactionOutputFileCount_95th_percentile": 1, "MajorCompactionOutputFileCount_98th_percentile": 1, "MajorCompactionOutputFileCount_99th_percentile": 1, "MajorCompactionOutputFileCount_99.9th_percentile": 1, "slowGetCount": 0, "ScanSize_num_ops": 9, "ScanSize_min": 0, "ScanSize_max": 1112, "ScanSize_mean": 889, "ScanSize_25th_percentile": 308, "ScanSize_median": 576, "ScanSize_75th_percentile": 844, "ScanSize_90th_percentile": 1004, "ScanSize_95th_percentile": 1058, "ScanSize_98th_percentile": 1090, "ScanSize_99th_percentile": 1101, "ScanSize_99.9th_percentile": 1110, "ScanSize_SizeRangeCount_0-10": 1, "ScanSize_SizeRangeCount_100-1000": 4, "CompactionOutputSize_num_ops": 2, "CompactionOutputSize_min": 4592, "CompactionOutputSize_max": 4626, "CompactionOutputSize_mean": 4609, "CompactionOutputSize_25th_percentile": 4600, "CompactionOutputSize_median": 4609, "CompactionOutputSize_75th_percentile": 4617, "CompactionOutputSize_90th_percentile": 4622, "CompactionOutputSize_95th_percentile": 4624, "CompactionOutputSize_98th_percentile": 4625, "CompactionOutputSize_99th_percentile": 4625, "CompactionOutputSize_99.9th_percentile": 4625, "CompactionOutputSize_SizeRangeCount_100-1000": 2, "PauseTimeWithoutGc_num_ops": 0, "PauseTimeWithoutGc_min": 0, "PauseTimeWithoutGc_max": 0, "PauseTimeWithoutGc_mean": 0, "PauseTimeWithoutGc_25th_percentile": 0, "PauseTimeWithoutGc_median": 0, "PauseTimeWithoutGc_75th_percentile": 0, "PauseTimeWithoutGc_90th_percentile": 0, "PauseTimeWithoutGc_95th_percentile": 0, "PauseTimeWithoutGc_98th_percentile": 0, "PauseTimeWithoutGc_99th_percentile": 0, "PauseTimeWithoutGc_99.9th_percentile": 0, "slowIncrementCount": 0, "Append_num_ops": 0, "Append_min": 0, "Append_max": 0, "Append_mean": 0, "Append_25th_percentile": 0, "Append_median": 0, "Append_75th_percentile": 0, "Append_90th_percentile": 0, "Append_95th_percentile": 0, "Append_98th_percentile": 0, "Append_99th_percentile": 0, "Append_99.9th_percentile": 0, "Bulkload_count": 0, "Bulkload_mean_rate": 0.0, "Bulkload_1min_rate": 0.0, "Bulkload_5min_rate": 0.0, "Bulkload_15min_rate": 0.0, "Bulkload_num_ops": 0, "Bulkload_min": 0, "Bulkload_max": 0, "Bulkload_mean": 0, "Bulkload_25th_percentile": 0, "Bulkload_median": 0, "Bulkload_75th_percentile": 0, "Bulkload_90th_percentile": 0, "Bulkload_95th_percentile": 0, "Bulkload_98th_percentile": 0, "Bulkload_99th_percentile": 0, "Bulkload_99.9th_percentile": 0 } ] } 2024-12-03T05:26:01,143 WARN [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=37365 {}] master.MasterRpcServices(700): e3c6570475b6,46191,1733203534664 reported a fatal error: ***** ABORTING region server e3c6570475b6,46191,1733203534664: testing ***** 2024-12-03T05:26:01,145 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e3c6570475b6,46191,1733203534664' ***** 2024-12-03T05:26:01,145 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: testing 2024-12-03T05:26:01,146 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T05:26:01,146 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T05:26:01,147 INFO [RS:2;e3c6570475b6:46191 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager abruptly. 2024-12-03T05:26:01,147 INFO [RS:2;e3c6570475b6:46191 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager abruptly. 2024-12-03T05:26:01,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=35421 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Get size: 140 connection: 172.17.0.3:48830 deadline: 1733203621146, exception=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=46191 startCode=1733203534664. As of locationSeqNum=12. 2024-12-03T05:26:01,147 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(3091): Received CLOSE for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,147 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(956): aborting server e3c6570475b6,46191,1733203534664 2024-12-03T05:26:01,147 INFO [RS:2;e3c6570475b6:46191 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:26:01,147 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=5 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=5, error=org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=46191 startCode=1733203534664. As of locationSeqNum=12. 2024-12-03T05:26:01,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=5 is org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=46191 startCode=1733203534664. As of locationSeqNum=12. 2024-12-03T05:26:01,148 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:26:01,148 INFO [RS:2;e3c6570475b6:46191 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:2;e3c6570475b6:46191. 2024-12-03T05:26:01,148 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,148 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(84): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=5 with the new location region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=12 constructed by org.apache.hadoop.hbase.exceptions.RegionMovedException: Region moved to: hostname=e3c6570475b6 port=46191 startCode=1733203534664. As of locationSeqNum=12. 2024-12-03T05:26:01,148 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,148 DEBUG [RS:2;e3c6570475b6:46191 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:26:01,148 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:26:01,148 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,148 DEBUG [RS:2;e3c6570475b6:46191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:01,149 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-03T05:26:01,149 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1325): Online Regions={3452b8a301e7ba272dae5f6efd7fe4dd=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.} 2024-12-03T05:26:01,149 DEBUG [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1351): Waiting on 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:01,151 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,151 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203561147Running coprocessor pre-close hooks at 1733203561147Disabling compacts and flushes for region at 1733203561147Disabling writes for close at 1733203561148 (+1 ms)Writing region close event to WAL at 1733203561151 (+3 ms)Running coprocessor post-close hooks at 1733203561151Closed at 1733203561151 2024-12-03T05:26:01,151 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:01,235 INFO [regionserver/e3c6570475b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:01,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e3c6570475b6,46191,1733203534664 aborting at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processRequest(ServerRpcConnection.java:564) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.ServerRpcConnection.processOneRpc(ServerRpcConnection.java:364) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyServerRpcConnection.process(NettyServerRpcConnection.java:89) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:56) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcServerRequestDecoder.channelRead0(NettyRpcServerRequestDecoder.java:31) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.SimpleChannelInboundHandler.channelRead(SimpleChannelInboundHandler.java:99) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.AbstractEpollStreamChannel$EpollStreamUnsafe.epollInReady(AbstractEpollStreamChannel.java:799) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.processReady(EpollEventLoop.java:501) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:399) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:01,260 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=12 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=12, error=org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e3c6570475b6,46191,1733203534664 aborting 2024-12-03T05:26:01,260 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=12 is org.apache.hadoop.hbase.regionserver.RegionServerAbortedException: Server e3c6570475b6,46191,1733203534664 aborting 2024-12-03T05:26:01,260 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=12 from cache 2024-12-03T05:26:01,349 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(976): stopping server e3c6570475b6,46191,1733203534664; all regions closed. 2024-12-03T05:26:01,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741834_1010 (size=2288) 2024-12-03T05:26:01,356 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664/e3c6570475b6%2C46191%2C1733203534664.1733203536948 not finished, retry = 0 2024-12-03T05:26:01,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741834_1010 (size=2288) 2024-12-03T05:26:01,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741834_1010 (size=2288) 2024-12-03T05:26:01,457 DEBUG [RS:2;e3c6570475b6:46191 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:01,457 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:01,457 INFO [RS:2;e3c6570475b6:46191 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:26:01,457 INFO [RS:2;e3c6570475b6:46191 {}] hbase.ChoreService(370): Chore service for: regionserver/e3c6570475b6:0 had [ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T05:26:01,458 INFO [regionserver/e3c6570475b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:26:01,458 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T05:26:01,458 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T05:26:01,458 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T05:26:01,458 INFO [RS:2;e3c6570475b6:46191 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:26:01,458 INFO [RS:2;e3c6570475b6:46191 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46191 2024-12-03T05:26:01,471 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18] 2024-12-03T05:26:01,472 WARN [RPCClient-NioEventLoopGroup-6-5 {}] ipc.NettyRpcConnection$2(409): Exception encountered while connecting to the server e3c6570475b6:46191 org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e3c6570475b6/172.17.0.3:46191 Caused by: java.net.ConnectException: Connection refused at sun.nio.ch.Net.pollConnect(Native Method) ~[?:?] at sun.nio.ch.Net.pollConnectNow(Net.java:672) ~[?:?] at sun.nio.ch.SocketChannelImpl.finishConnect(SocketChannelImpl.java:946) ~[?:?] at org.apache.hbase.thirdparty.io.netty.channel.socket.nio.NioSocketChannel.doFinishConnect(NioSocketChannel.java:336) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioChannel$AbstractNioUnsafe.finishConnect(AbstractNioChannel.java:339) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:776) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:01,473 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18, error=java.net.ConnectException: Call to address=e3c6570475b6:46191 failed on connection exception: org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e3c6570475b6/172.17.0.3:46191 2024-12-03T05:26:01,473 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 is java.net.ConnectException: Connection refused 2024-12-03T05:26:01,473 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 from cache 2024-12-03T05:26:01,473 DEBUG [RPCClient-NioEventLoopGroup-6-5 {}] ipc.FailedServers(52): Added failed server with address e3c6570475b6:46191 to list caused by org.apache.hbase.thirdparty.io.netty.channel.AbstractChannel$AnnotatedConnectException: Connection refused: e3c6570475b6/172.17.0.3:46191 2024-12-03T05:26:01,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:26:01,507 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e3c6570475b6,46191,1733203534664 2024-12-03T05:26:01,507 INFO [RS:2;e3c6570475b6:46191 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:26:01,525 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e3c6570475b6,46191,1733203534664] 2024-12-03T05:26:01,535 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e3c6570475b6,46191,1733203534664 already deleted, retry=false 2024-12-03T05:26:01,536 INFO [RegionServerTracker-0 {}] master.ServerManager(695): Processing expiration of e3c6570475b6,46191,1733203534664 on e3c6570475b6,37365,1733203533327 2024-12-03T05:26:01,540 DEBUG [RegionServerTracker-0 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=false; ServerCrashProcedure e3c6570475b6,46191,1733203534664, splitWal=true, meta=false 2024-12-03T05:26:01,543 INFO [RegionServerTracker-0 {}] assignment.AssignmentManager(1999): Scheduled ServerCrashProcedure pid=13 for e3c6570475b6,46191,1733203534664 (carryingMeta=false) e3c6570475b6,46191,1733203534664/CRASHED/regionCount=1/lock=java.util.concurrent.locks.ReentrantReadWriteLock@22f65029[Write locks = 1, Read locks = 0], oldState=ONLINE. 2024-12-03T05:26:01,543 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(169): Start pid=13, state=RUNNABLE:SERVER_CRASH_START, hasLock=true; ServerCrashProcedure e3c6570475b6,46191,1733203534664, splitWal=true, meta=false 2024-12-03T05:26:01,545 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(207): e3c6570475b6,46191,1733203534664 had 1 regions 2024-12-03T05:26:01,546 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(339): Splitting WALs pid=13, state=RUNNABLE:SERVER_CRASH_SPLIT_LOGS, hasLock=true; ServerCrashProcedure e3c6570475b6,46191,1733203534664, splitWal=true, meta=false, isMeta: false 2024-12-03T05:26:01,548 DEBUG [PEWorker-5 {}] master.MasterWalManager(329): Renamed region directory: hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting 2024-12-03T05:26:01,549 INFO [PEWorker-5 {}] master.SplitWALManager(105): e3c6570475b6,46191,1733203534664 WAL count=1, meta=false 2024-12-03T05:26:01,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE:ACQUIRE_SPLIT_WAL_WORKER, hasLock=false; SplitWALProcedure e3c6570475b6%2C46191%2C1733203534664.1733203536948}] 2024-12-03T05:26:01,557 DEBUG [PEWorker-1 {}] master.SplitWALManager(158): Acquired split WAL worker=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:01,560 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=15, ppid=14, state=RUNNABLE, hasLock=false; SplitWALRemoteProcedure e3c6570475b6%2C46191%2C1733203534664.1733203536948, worker=e3c6570475b6,35421,1733203534454}] 2024-12-03T05:26:01,625 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:01,626 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46191-0x10199a715d70003, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:01,626 INFO [RS:2;e3c6570475b6:46191 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:26:01,626 INFO [RS:2;e3c6570475b6:46191 {}] regionserver.HRegionServer(1031): Exiting; stopping=e3c6570475b6,46191,1733203534664; zookeeper connection closed. 2024-12-03T05:26:01,626 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@5d929c92 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@5d929c92 2024-12-03T05:26:01,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=35421 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.SplitWALCallable, pid=15 2024-12-03T05:26:01,744 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948, size=2.2 K (2288bytes) 2024-12-03T05:26:01,744 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 2024-12-03T05:26:01,745 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 after 1ms 2024-12-03T05:26:01,781 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:01,781 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(310): Open hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 took 37ms 2024-12-03T05:26:01,782 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18] 2024-12-03T05:26:01,783 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] ipc.AbstractRpcClient(357): Not trying to connect to e3c6570475b6:46191 this server is in the failed servers list 2024-12-03T05:26:01,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 , the old value is region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18, error=org.apache.hadoop.hbase.ipc.FailedServerException: Call to address=e3c6570475b6:46191 failed on local exception: org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e3c6570475b6:46191 2024-12-03T05:26:01,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 is org.apache.hadoop.hbase.ipc.FailedServerException: This server is in the failed servers list: e3c6570475b6:46191 2024-12-03T05:26:01,784 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncRegionLocatorHelper(88): Try removing region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,46191,1733203534664, seqNum=18 from cache 2024-12-03T05:26:01,790 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(352): Last flushed sequenceid for 3452b8a301e7ba272dae5f6efd7fe4dd: last_flushed_sequence_id: 12 store_sequence_id { family_name: "cf1" sequence_id: 12 } store_sequence_id { family_name: "cf2" sequence_id: 12 } 2024-12-03T05:26:01,790 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 so closing down 2024-12-03T05:26:01,790 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:01,791 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:01,791 INFO [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(425): Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948, size=2.2 K, length=2288, corrupted=false, cancelled=false 2024-12-03T05:26:01,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948, journal: Splitting hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948, size=2.2 K (2288bytes) at 1733203561744Finishing writing output for hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 so closing down at 1733203561790 (+46 ms)3 split writer threads finished at 1733203561791 (+1 ms)Processed 6 edits across 0 Regions in 9 ms; skipped=6; WAL=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948, size=2.2 K, length=2288, corrupted=false, cancelled=false at 1733203561791 2024-12-03T05:26:01,791 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.SplitLogWorker(218): Done splitting WAL hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 2024-12-03T05:26:01,792 DEBUG [RS_LOG_REPLAY_OPS-regionserver/e3c6570475b6:0-0 {event_type=RS_LOG_REPLAY, pid=15}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=15 2024-12-03T05:26:01,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=37365 {}] master.HMaster(4169): Remote procedure done, pid=15 2024-12-03T05:26:01,799 INFO [PEWorker-3 {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting/e3c6570475b6%2C46191%2C1733203534664.1733203536948 to hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs 2024-12-03T05:26:01,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=15, resume processing ppid=14 2024-12-03T05:26:01,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, ppid=14, state=SUCCESS, hasLock=false; SplitWALRemoteProcedure e3c6570475b6%2C46191%2C1733203534664.1733203536948, worker=e3c6570475b6,35421,1733203534454 in 240 msec 2024-12-03T05:26:01,803 DEBUG [PEWorker-2 {}] master.SplitWALManager(172): Release split WAL worker=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:01,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-03T05:26:01,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; SplitWALProcedure e3c6570475b6%2C46191%2C1733203534664.1733203536948, worker=e3c6570475b6,35421,1733203534454 in 254 msec 2024-12-03T05:26:01,809 INFO [PEWorker-5 {}] master.SplitLogManager(171): hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting dir is empty, no logs to split. 2024-12-03T05:26:01,809 INFO [PEWorker-5 {}] master.SplitWALManager(105): e3c6570475b6,46191,1733203534664 WAL count=0, meta=false 2024-12-03T05:26:01,809 DEBUG [PEWorker-5 {}] procedure.ServerCrashProcedure(329): Check if e3c6570475b6,46191,1733203534664 WAL splitting is done? wals=0, meta=false 2024-12-03T05:26:01,811 INFO [PEWorker-5 {}] procedure.ServerCrashProcedure(321): Remove WAL directory for e3c6570475b6,46191,1733203534664 failed, ignore...File hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/WALs/e3c6570475b6,46191,1733203534664-splitting does not exist. 2024-12-03T05:26:01,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN}] 2024-12-03T05:26:01,817 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN 2024-12-03T05:26:01,818 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN; state=OPEN, location=null; forceNewPlan=true, retain=false 2024-12-03T05:26:01,969 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(204): Hosts are {e3c6570475b6=0} racks are {/default-rack=0} 2024-12-03T05:26:01,969 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(303): server 0 has 0 regions 2024-12-03T05:26:01,969 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(303): server 1 has 0 regions 2024-12-03T05:26:01,969 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(310): server 0 is on host 0 2024-12-03T05:26:01,969 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(310): server 1 is on host 0 2024-12-03T05:26:01,969 INFO [e3c6570475b6:37365 {}] balancer.BalancerClusterState(321): server 0 is on rack 0 2024-12-03T05:26:01,969 INFO [e3c6570475b6:37365 {}] balancer.BalancerClusterState(321): server 1 is on rack 0 2024-12-03T05:26:01,970 DEBUG [e3c6570475b6:37365 {}] balancer.BalancerClusterState(326): Number of tables=1, number of hosts=1, number of racks=1 2024-12-03T05:26:01,970 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPENING, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:01,973 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=16, ppid=13, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN because future has completed 2024-12-03T05:26:01,974 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454}] 2024-12-03T05:26:02,132 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(132): Open testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:02,132 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7752): Opening region: {ENCODED => 3452b8a301e7ba272dae5f6efd7fe4dd, NAME => 'testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:02,132 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterRegionMovedWithMultiCF 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,132 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(898): Instantiated testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:02,133 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7794): checking encryption for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,133 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(7797): checking classloading for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,134 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf1 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,135 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf1 2024-12-03T05:26:02,136 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,143 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf1/c24776419397411c9d72d0a34af9e29a 2024-12-03T05:26:02,144 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf1, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,144 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family cf2 of region 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,145 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3452b8a301e7ba272dae5f6efd7fe4dd columnFamilyName cf2 2024-12-03T05:26:02,145 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,153 DEBUG [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/cf2/366f5a163ef54fb892f547d987986a41 2024-12-03T05:26:02,153 INFO [StoreOpener-3452b8a301e7ba272dae5f6efd7fe4dd-1 {}] regionserver.HStore(327): Store=3452b8a301e7ba272dae5f6efd7fe4dd/cf2, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,153 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1038): replaying wal for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,155 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,156 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,157 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1048): stopping wal replay for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,157 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1060): Cleaning up temporary data for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,158 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterRegionMovedWithMultiCF descriptor;using region.getMemStoreFlushHeapSize/# of families (64.0 M)) instead. 2024-12-03T05:26:02,159 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1093): writing seq id for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,161 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1114): Opened 3452b8a301e7ba272dae5f6efd7fe4dd; next sequenceid=18; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63959613, jitterRate=-0.046927496790885925}}}, FlushLargeStoresPolicy{flushSizeLowerBound=67108864} 2024-12-03T05:26:02,161 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:02,162 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegion(1006): Region open journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Running coprocessor pre-open hook at 1733203562133Writing region info on filesystem at 1733203562133Initializing all the Stores at 1733203562134 (+1 ms)Instantiating store for column family {NAME => 'cf1', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562134Instantiating store for column family {NAME => 'cf2', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'NONE', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562134Cleaning up temporary data from old regions at 1733203562157 (+23 ms)Running coprocessor post-open hooks at 1733203562161 (+4 ms)Region opened successfully at 1733203562162 (+1 ms) 2024-12-03T05:26:02,163 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2236): Post open deploy tasks for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., pid=17, masterSystemTime=1733203562127 2024-12-03T05:26:02,165 DEBUG [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] regionserver.HRegionServer(2266): Finished post open deploy task for testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:02,166 INFO [RS_OPEN_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_OPEN_REGION, pid=17}] handler.AssignRegionHandler(153): Opened testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:02,166 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=16 updating hbase:meta row=3452b8a301e7ba272dae5f6efd7fe4dd, regionState=OPEN, openSeqNum=18, regionLocation=e3c6570475b6,35421,1733203534454 2024-12-03T05:26:02,169 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-5-2 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=17, ppid=16, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 because future has completed 2024-12-03T05:26:02,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=17, resume processing ppid=16 2024-12-03T05:26:02,173 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, ppid=16, state=SUCCESS, hasLock=false; OpenRegionProcedure 3452b8a301e7ba272dae5f6efd7fe4dd, server=e3c6570475b6,35421,1733203534454 in 196 msec 2024-12-03T05:26:02,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=13 2024-12-03T05:26:02,176 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=13, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=testReplayEditsAfterRegionMovedWithMultiCF, region=3452b8a301e7ba272dae5f6efd7fe4dd, ASSIGN in 358 msec 2024-12-03T05:26:02,176 INFO [PEWorker-3 {}] procedure.ServerCrashProcedure(291): removed crashed server e3c6570475b6,46191,1733203534664 after splitting done 2024-12-03T05:26:02,180 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; ServerCrashProcedure e3c6570475b6,46191,1733203534664, splitWal=true, meta=false in 640 msec 2024-12-03T05:26:02,253 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-03T05:26:02,301 DEBUG [RPCClient-NioEventLoopGroup-6-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'testReplayEditsAfterRegionMovedWithMultiCF', row='r1', locateType=CURRENT is [region=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., hostname=e3c6570475b6,35421,1733203534454, seqNum=18] 2024-12-03T05:26:02,320 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterRegionMovedWithMultiCF Thread=402 (was 413), OpenFileDescriptor=1043 (was 1023) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=705 (was 671) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3212 (was 3575) 2024-12-03T05:26:02,323 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-12-03T05:26:02,338 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=402, OpenFileDescriptor=1043, MaxFileDescriptor=1048576, SystemLoadAverage=705, ProcessCount=11, AvailableMemoryMB=3207 2024-12-03T05:26:02,338 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1043 is superior to 1024 2024-12-03T05:26:02,360 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:02,362 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:02,364 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:26:02,367 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-64475552, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-64475552, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:02,383 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-64475552/hregion-64475552.1733203562367, exclude list is [], retry=0 2024-12-03T05:26:02,386 DEBUG [AsyncFSWAL-20-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:02,386 DEBUG [AsyncFSWAL-20-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:02,387 DEBUG [AsyncFSWAL-20-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:02,430 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-64475552/hregion-64475552.1733203562367 2024-12-03T05:26:02,431 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:26:02,431 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => f2e931f2d123916e267aaa301fa14077, NAME => 'testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:26:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741893_1071 (size=67) 2024-12-03T05:26:02,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741893_1071 (size=67) 2024-12-03T05:26:02,443 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741893_1071 (size=67) 2024-12-03T05:26:02,443 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:02,445 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,446 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName a 2024-12-03T05:26:02,446 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,447 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,447 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,448 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName b 2024-12-03T05:26:02,449 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,449 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,449 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,450 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName c 2024-12-03T05:26:02,451 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,451 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,451 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,452 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,453 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,453 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,454 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:02,455 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,463 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:26:02,463 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2e931f2d123916e267aaa301fa14077; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69704370, jitterRate=0.038676053285598755}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2e931f2d123916e267aaa301fa14077: Writing region info on filesystem at 1733203562443Initializing all the Stores at 1733203562444 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562444Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562444Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562444Cleaning up temporary data from old regions at 1733203562453 (+9 ms)Region opened successfully at 1733203562464 (+11 ms) 2024-12-03T05:26:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2e931f2d123916e267aaa301fa14077, disabling compactions & flushes 2024-12-03T05:26:02,464 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. after waiting 0 ms 2024-12-03T05:26:02,464 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,465 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,465 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2e931f2d123916e267aaa301fa14077: Waiting for close lock at 1733203562464Disabling compacts and flushes for region at 1733203562464Disabling writes for close at 1733203562464Writing region close event to WAL at 1733203562465 (+1 ms)Closed at 1733203562465 2024-12-03T05:26:02,469 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-64475552/hregion-64475552.1733203562367 not finished, retry = 0 2024-12-03T05:26:02,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741892_1070 (size=175) 2024-12-03T05:26:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741892_1070 (size=175) 2024-12-03T05:26:02,470 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741892_1070 (size=175) 2024-12-03T05:26:02,572 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:02,572 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-64475552:(num 1733203562367) 2024-12-03T05:26:02,572 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:02,574 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:02,587 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, exclude list is [], retry=0 2024-12-03T05:26:02,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:02,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:02,590 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:02,614 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 2024-12-03T05:26:02,614 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:02,615 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2e931f2d123916e267aaa301fa14077, NAME => 'testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:02,615 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:02,615 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,615 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,616 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,617 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName a 2024-12-03T05:26:02,617 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,617 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,618 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,618 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName b 2024-12-03T05:26:02,618 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,619 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,619 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,619 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName c 2024-12-03T05:26:02,619 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,620 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,620 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,621 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,622 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,623 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,623 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,623 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:02,624 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,625 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2e931f2d123916e267aaa301fa14077; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59323569, jitterRate=-0.11600993573665619}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:02,626 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2e931f2d123916e267aaa301fa14077: Writing region info on filesystem at 1733203562615Initializing all the Stores at 1733203562616 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562616Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562616Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562616Cleaning up temporary data from old regions at 1733203562623 (+7 ms)Region opened successfully at 1733203562626 (+3 ms) 2024-12-03T05:26:02,650 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2e931f2d123916e267aaa301fa14077 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T05:26:02,664 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/a/4d9ad9cf64a24b50b7297e7add569d41 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733203562626/Put/seqid=0 2024-12-03T05:26:02,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741895_1073 (size=5958) 2024-12-03T05:26:02,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741895_1073 (size=5958) 2024-12-03T05:26:02,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741895_1073 (size=5958) 2024-12-03T05:26:02,671 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/a/4d9ad9cf64a24b50b7297e7add569d41 2024-12-03T05:26:02,688 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/6910c14bf9bc41b3abbc8868bf9a2397 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733203562635/Put/seqid=0 2024-12-03T05:26:02,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741896_1074 (size=5958) 2024-12-03T05:26:02,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741896_1074 (size=5958) 2024-12-03T05:26:02,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741896_1074 (size=5958) 2024-12-03T05:26:02,696 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/6910c14bf9bc41b3abbc8868bf9a2397 2024-12-03T05:26:02,714 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/c/63c45ef6c24047d98e92fc1e780ad882 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733203562641/Put/seqid=0 2024-12-03T05:26:02,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741897_1075 (size=5958) 2024-12-03T05:26:02,732 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741897_1075 (size=5958) 2024-12-03T05:26:02,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741897_1075 (size=5958) 2024-12-03T05:26:02,733 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=33 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/c/63c45ef6c24047d98e92fc1e780ad882 2024-12-03T05:26:02,742 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/a/4d9ad9cf64a24b50b7297e7add569d41 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/a/4d9ad9cf64a24b50b7297e7add569d41 2024-12-03T05:26:02,749 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/a/4d9ad9cf64a24b50b7297e7add569d41, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T05:26:02,755 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/6910c14bf9bc41b3abbc8868bf9a2397 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/6910c14bf9bc41b3abbc8868bf9a2397 2024-12-03T05:26:02,761 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/6910c14bf9bc41b3abbc8868bf9a2397, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T05:26:02,763 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/c/63c45ef6c24047d98e92fc1e780ad882 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/c/63c45ef6c24047d98e92fc1e780ad882 2024-12-03T05:26:02,769 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/c/63c45ef6c24047d98e92fc1e780ad882, entries=10, sequenceid=33, filesize=5.8 K 2024-12-03T05:26:02,771 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for f2e931f2d123916e267aaa301fa14077 in 120ms, sequenceid=33, compaction requested=false 2024-12-03T05:26:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for f2e931f2d123916e267aaa301fa14077: 2024-12-03T05:26:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing f2e931f2d123916e267aaa301fa14077, disabling compactions & flushes 2024-12-03T05:26:02,771 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. after waiting 0 ms 2024-12-03T05:26:02,771 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,773 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. 2024-12-03T05:26:02,773 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for f2e931f2d123916e267aaa301fa14077: Waiting for close lock at 1733203562771Disabling compacts and flushes for region at 1733203562771Disabling writes for close at 1733203562771Writing region close event to WAL at 1733203562773 (+2 ms)Closed at 1733203562773 2024-12-03T05:26:02,779 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741894_1072 (size=6311) 2024-12-03T05:26:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741894_1072 (size=6311) 2024-12-03T05:26:02,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741894_1072 (size=6311) 2024-12-03T05:26:02,784 DEBUG [Time-limited test {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/6910c14bf9bc41b3abbc8868bf9a2397 to hdfs://localhost:43427/hbase/archive/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/6910c14bf9bc41b3abbc8868bf9a2397 2024-12-03T05:26:02,807 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, size=6.2 K (6311bytes) 2024-12-03T05:26:02,807 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 2024-12-03T05:26:02,807 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 after 0ms 2024-12-03T05:26:02,833 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:02,833 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 took 27ms 2024-12-03T05:26:02,836 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 so closing down 2024-12-03T05:26:02,836 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:02,837 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733203562574.temp 2024-12-03T05:26:02,864 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp 2024-12-03T05:26:02,865 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:02,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741898_1076 (size=5575) 2024-12-03T05:26:02,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741898_1076 (size=5575) 2024-12-03T05:26:02,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741898_1076 (size=5575) 2024-12-03T05:26:02,879 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:02,881 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 2024-12-03T05:26:02,882 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, size=6.2 K, length=6311, corrupted=false, cancelled=false 2024-12-03T05:26:02,882 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, journal: Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, size=6.2 K (6311bytes) at 1733203562807Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 so closing down at 1733203562836 (+29 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp at 1733203562865 (+29 ms)3 split writer threads finished at 1733203562865Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733203562879 (+14 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000003-wal.1733203562574.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 at 1733203562881 (+2 ms)Processed 32 edits across 1 Regions in 48 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574, size=6.2 K, length=6311, corrupted=false, cancelled=false at 1733203562882 (+1 ms) 2024-12-03T05:26:02,884 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562574 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203562574 2024-12-03T05:26:02,885 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 2024-12-03T05:26:02,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:02,887 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:02,899 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562887, exclude list is [], retry=0 2024-12-03T05:26:02,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:02,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:02,902 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:02,941 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562887 2024-12-03T05:26:02,942 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:26:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => f2e931f2d123916e267aaa301fa14077, NAME => 'testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,942 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,944 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,945 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName a 2024-12-03T05:26:02,945 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,950 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/a/4d9ad9cf64a24b50b7297e7add569d41 2024-12-03T05:26:02,950 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,950 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,952 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName b 2024-12-03T05:26:02,952 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,953 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,953 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,953 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f2e931f2d123916e267aaa301fa14077 columnFamilyName c 2024-12-03T05:26:02,954 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:02,961 DEBUG [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/c/63c45ef6c24047d98e92fc1e780ad882 2024-12-03T05:26:02,961 INFO [StoreOpener-f2e931f2d123916e267aaa301fa14077-1 {}] regionserver.HStore(327): Store=f2e931f2d123916e267aaa301fa14077/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:02,962 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,963 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,965 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:02,966 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 2024-12-03T05:26:02,993 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:02,994 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 20, firstSequenceIdInLog=3, maxSequenceIdInLog=32, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 2024-12-03T05:26:02,995 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing f2e931f2d123916e267aaa301fa14077 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T05:26:03,013 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/b681f88c5875496c993184aa0483bc82 is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733203562635/Put/seqid=0 2024-12-03T05:26:03,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741900_1078 (size=5958) 2024-12-03T05:26:03,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741900_1078 (size=5958) 2024-12-03T05:26:03,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741900_1078 (size=5958) 2024-12-03T05:26:03,028 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=32 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/b681f88c5875496c993184aa0483bc82 2024-12-03T05:26:03,037 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/.tmp/b/b681f88c5875496c993184aa0483bc82 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/b681f88c5875496c993184aa0483bc82 2024-12-03T05:26:03,048 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/b/b681f88c5875496c993184aa0483bc82, entries=10, sequenceid=32, filesize=5.8 K 2024-12-03T05:26:03,049 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f2e931f2d123916e267aaa301fa14077 in 54ms, sequenceid=32, compaction requested=false; wal=null 2024-12-03T05:26:03,050 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/0000000000000000032 2024-12-03T05:26:03,051 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:03,051 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:03,052 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:03,054 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for f2e931f2d123916e267aaa301fa14077 2024-12-03T05:26:03,056 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/f2e931f2d123916e267aaa301fa14077/recovered.edits/33.seqid, newMaxSeqId=33, maxSeqId=1 2024-12-03T05:26:03,057 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened f2e931f2d123916e267aaa301fa14077; next sequenceid=34; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68357490, jitterRate=0.018605977296829224}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:03,058 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for f2e931f2d123916e267aaa301fa14077: Writing region info on filesystem at 1733203562942Initializing all the Stores at 1733203562943 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562943Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562944 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203562944Obtaining lock to block concurrent updates at 1733203562995 (+51 ms)Preparing flush snapshotting stores in f2e931f2d123916e267aaa301fa14077 at 1733203562995Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077., syncing WAL and waiting on mvcc, flushsize=dataSize=870, getHeapSize=2320, getOffHeapSize=0, getCellsCount=10 at 1733203562995Flushing stores of testReplayEditsWrittenViaHRegion,,1733203562361.f2e931f2d123916e267aaa301fa14077. at 1733203562995Flushing f2e931f2d123916e267aaa301fa14077/b: creating writer at 1733203562995Flushing f2e931f2d123916e267aaa301fa14077/b: appending metadata at 1733203563012 (+17 ms)Flushing f2e931f2d123916e267aaa301fa14077/b: closing flushed file at 1733203563012Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7ce08c0c: reopening flushed file at 1733203563036 (+24 ms)Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for f2e931f2d123916e267aaa301fa14077 in 54ms, sequenceid=32, compaction requested=false; wal=null at 1733203563049 (+13 ms)Cleaning up temporary data from old regions at 1733203563051 (+2 ms)Region opened successfully at 1733203563058 (+7 ms) 2024-12-03T05:26:03,085 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterPartialFlush Thread=411 (was 402) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:48512 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47016 [Waiting for operation #32] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-20-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57280 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:52664 [Waiting for operation #40] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:52192 [Waiting for operation #30] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:48832 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741899_1077, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1113 (was 1043) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=705 (was 705), ProcessCount=11 (was 11), AvailableMemoryMB=3130 (was 3207) 2024-12-03T05:26:03,085 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1113 is superior to 1024 2024-12-03T05:26:03,096 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=411, OpenFileDescriptor=1113, MaxFileDescriptor=1048576, SystemLoadAverage=705, ProcessCount=11, AvailableMemoryMB=3130 2024-12-03T05:26:03,096 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1113 is superior to 1024 2024-12-03T05:26:03,109 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:03,111 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:03,111 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:26:03,114 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-99323407, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-99323407, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:03,133 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-99323407/hregion-99323407.1733203563115, exclude list is [], retry=0 2024-12-03T05:26:03,136 DEBUG [AsyncFSWAL-22-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:03,139 DEBUG [AsyncFSWAL-22-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:03,140 DEBUG [AsyncFSWAL-22-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:03,180 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-99323407/hregion-99323407.1733203563115 2024-12-03T05:26:03,180 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:03,180 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e82ebda03d2152966c59ec19cd109b2d, NAME => 'testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsAfterAbortingFlush', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:26:03,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741902_1080 (size=68) 2024-12-03T05:26:03,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741902_1080 (size=68) 2024-12-03T05:26:03,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741902_1080 (size=68) 2024-12-03T05:26:03,191 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:03,194 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,196 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName a 2024-12-03T05:26:03,196 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,197 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,197 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,198 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName b 2024-12-03T05:26:03,198 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,199 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,199 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,201 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName c 2024-12-03T05:26:03,201 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,202 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,202 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,203 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,205 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,205 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,206 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:03,207 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,210 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:26:03,210 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e82ebda03d2152966c59ec19cd109b2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68972690, jitterRate=0.027773171663284302}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e82ebda03d2152966c59ec19cd109b2d: Writing region info on filesystem at 1733203563192Initializing all the Stores at 1733203563192Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563192Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563193 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563193Cleaning up temporary data from old regions at 1733203563205 (+12 ms)Region opened successfully at 1733203563211 (+6 ms) 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e82ebda03d2152966c59ec19cd109b2d, disabling compactions & flushes 2024-12-03T05:26:03,211 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. after waiting 0 ms 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:03,211 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:03,211 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e82ebda03d2152966c59ec19cd109b2d: Waiting for close lock at 1733203563211Disabling compacts and flushes for region at 1733203563211Disabling writes for close at 1733203563211Writing region close event to WAL at 1733203563211Closed at 1733203563211 2024-12-03T05:26:03,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741901_1079 (size=175) 2024-12-03T05:26:03,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741901_1079 (size=175) 2024-12-03T05:26:03,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741901_1079 (size=175) 2024-12-03T05:26:03,219 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:03,219 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-99323407:(num 1733203563115) 2024-12-03T05:26:03,219 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:03,221 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:03,233 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, exclude list is [], retry=0 2024-12-03T05:26:03,236 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:03,237 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:03,237 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:03,264 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 2024-12-03T05:26:03,265 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:26:03,339 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e82ebda03d2152966c59ec19cd109b2d, NAME => 'testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:03,341 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,342 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:03,342 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,342 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,344 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,345 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName a 2024-12-03T05:26:03,346 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,346 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,347 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,348 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName b 2024-12-03T05:26:03,348 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,349 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,349 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,350 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName c 2024-12-03T05:26:03,350 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:03,351 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:03,351 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,351 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,353 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,354 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,354 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,354 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:03,356 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,356 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e82ebda03d2152966c59ec19cd109b2d; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68739434, jitterRate=0.024297386407852173}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:03,357 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:03,357 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e82ebda03d2152966c59ec19cd109b2d: Running coprocessor pre-open hook at 1733203563342Writing region info on filesystem at 1733203563342Initializing all the Stores at 1733203563343 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563343Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563343Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203563343Cleaning up temporary data from old regions at 1733203563354 (+11 ms)Running coprocessor post-open hooks at 1733203563357 (+3 ms)Region opened successfully at 1733203563357 2024-12-03T05:26:03,384 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e82ebda03d2152966c59ec19cd109b2d 3/3 column families, dataSize=590 B heapSize=2.08 KB 2024-12-03T05:26:03,393 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=0 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:03,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion 2024-12-03T05:26:03,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsWrittenViaHRegion Metrics about Tables on a single HBase RegionServer 2024-12-03T05:26:03,831 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF 2024-12-03T05:26:03,831 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterRegionMovedWithMultiCF Metrics about Tables on a single HBase RegionServer 2024-12-03T05:26:03,832 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush 2024-12-03T05:26:03,832 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testReplayEditsAfterAbortingFlush Metrics about Tables on a single HBase RegionServer 2024-12-03T05:26:04,394 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=1 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:04,640 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:26:05,395 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=2 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:06,395 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=3 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:07,396 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=4 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:08,397 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=5 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:09,397 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=6 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:10,398 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=7 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:11,399 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=8 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:12,399 WARN [Time-limited test {}] regionserver.HStore(850): Failed flushing store file for e82ebda03d2152966c59ec19cd109b2d/a, retrying num=9 java.io.IOException: Simulated exception by tests at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay$CustomStoreFlusher.flushSnapshot(AbstractTestWALReplay.java:619) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.HStore.flushCache(HStore.java:832) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl.flushCache(HStore.java:1975) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushCacheAndCommit(HRegion.java:3029) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2737) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.internalFlushcache(HRegion.java:2709) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2579) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flushcache(HRegion.java:2502) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.flush(HRegion.java:2472) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.testReplayEditsAfterAbortingFlush(AbstractTestWALReplay.java:668) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:12,401 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e82ebda03d2152966c59ec19cd109b2d: 2024-12-03T05:26:12,401 INFO [Time-limited test {}] wal.AbstractTestWALReplay(671): Expected simulated exception when flushing region, region: testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,424 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e82ebda03d2152966c59ec19cd109b2d: 2024-12-03T05:26:12,424 INFO [Time-limited test {}] wal.AbstractTestWALReplay(691): Expected exception when flushing region because server is stopped,Aborting flush because server is aborted... 2024-12-03T05:26:12,424 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e82ebda03d2152966c59ec19cd109b2d, disabling compactions & flushes 2024-12-03T05:26:12,424 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,424 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,424 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. after waiting 0 ms 2024-12-03T05:26:12,424 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,425 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1190 in region testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,425 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. 2024-12-03T05:26:12,425 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e82ebda03d2152966c59ec19cd109b2d: Waiting for close lock at 1733203572424Running coprocessor pre-close hooks at 1733203572424Disabling compacts and flushes for region at 1733203572424Disabling writes for close at 1733203572424Writing region close event to WAL at 1733203572425 (+1 ms)Running coprocessor post-close hooks at 1733203572425Closed at 1733203572425 2024-12-03T05:26:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741903_1081 (size=4236) 2024-12-03T05:26:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741903_1081 (size=4236) 2024-12-03T05:26:12,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741903_1081 (size=4236) 2024-12-03T05:26:12,456 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, size=4.1 K (4236bytes) 2024-12-03T05:26:12,457 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 2024-12-03T05:26:12,458 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 after 1ms 2024-12-03T05:26:12,491 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:12,491 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 took 35ms 2024-12-03T05:26:12,493 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 so closing down 2024-12-03T05:26:12,493 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:12,494 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000004-wal.1733203563221.temp 2024-12-03T05:26:12,518 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp 2024-12-03T05:26:12,518 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:12,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741904_1082 (size=3245) 2024-12-03T05:26:12,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741904_1082 (size=3245) 2024-12-03T05:26:12,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741904_1082 (size=3245) 2024-12-03T05:26:12,526 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp (wrote 20 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:12,527 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 2024-12-03T05:26:12,527 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, size=4.1 K, length=4236, corrupted=false, cancelled=false 2024-12-03T05:26:12,527 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, journal: Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, size=4.1 K (4236bytes) at 1733203572457Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 so closing down at 1733203572493 (+36 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp at 1733203572518 (+25 ms)3 split writer threads finished at 1733203572518Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp (wrote 20 edits, skipped 0 edits in 0 ms) at 1733203572526 (+8 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000004-wal.1733203563221.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 at 1733203572527 (+1 ms)Processed 23 edits across 1 Regions in 36 ms; skipped=3; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221, size=4.1 K, length=4236, corrupted=false, cancelled=false at 1733203572527 2024-12-03T05:26:12,529 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203563221 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203563221 2024-12-03T05:26:12,530 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 2024-12-03T05:26:12,530 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:12,532 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:12,544 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203572532, exclude list is [], retry=0 2024-12-03T05:26:12,547 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:12,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:12,548 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:12,573 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203572532 2024-12-03T05:26:12,573 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:26:12,573 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e82ebda03d2152966c59ec19cd109b2d, NAME => 'testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:12,574 DEBUG [Time-limited test {}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table testReplayEditsAfterAbortingFlush e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,574 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:12,574 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,574 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,576 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,577 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName a 2024-12-03T05:26:12,577 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:12,578 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:12,578 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,578 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName b 2024-12-03T05:26:12,579 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:12,579 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:12,579 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,580 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e82ebda03d2152966c59ec19cd109b2d columnFamilyName c 2024-12-03T05:26:12,580 DEBUG [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:12,581 INFO [StoreOpener-e82ebda03d2152966c59ec19cd109b2d-1 {}] regionserver.HStore(327): Store=e82ebda03d2152966c59ec19cd109b2d/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:12,581 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,582 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,585 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,586 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 2024-12-03T05:26:12,620 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:12,621 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 0, firstSequenceIdInLog=4, maxSequenceIdInLog=26, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 2024-12-03T05:26:12,621 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e82ebda03d2152966c59ec19cd109b2d 3/3 column families, dataSize=1.16 KB heapSize=3.41 KB 2024-12-03T05:26:12,636 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/a/5a38781bfe8d4be6bc03995b99cd733c is 64, key is testReplayEditsAfterAbortingFlush12/a:q/1733203572407/Put/seqid=0 2024-12-03T05:26:12,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741906_1084 (size=5523) 2024-12-03T05:26:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741906_1084 (size=5523) 2024-12-03T05:26:12,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741906_1084 (size=5523) 2024-12-03T05:26:12,642 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=416 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/a/5a38781bfe8d4be6bc03995b99cd733c 2024-12-03T05:26:12,667 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/b/479856e6b97142f9bd00d355919c177e is 64, key is testReplayEditsAfterAbortingFlush10/b:q/1733203572401/Put/seqid=0 2024-12-03T05:26:12,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741907_1085 (size=5524) 2024-12-03T05:26:12,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741907_1085 (size=5524) 2024-12-03T05:26:12,676 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741907_1085 (size=5524) 2024-12-03T05:26:12,676 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=417 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/b/479856e6b97142f9bd00d355919c177e 2024-12-03T05:26:12,702 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/c/8050c870e5134c298c5bde2deab65c92 is 64, key is testReplayEditsAfterAbortingFlush11/c:q/1733203572404/Put/seqid=0 2024-12-03T05:26:12,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741908_1086 (size=5457) 2024-12-03T05:26:12,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741908_1086 (size=5457) 2024-12-03T05:26:12,709 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=357 B at sequenceid=26 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/c/8050c870e5134c298c5bde2deab65c92 2024-12-03T05:26:12,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741908_1086 (size=5457) 2024-12-03T05:26:12,720 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/a/5a38781bfe8d4be6bc03995b99cd733c as hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/a/5a38781bfe8d4be6bc03995b99cd733c 2024-12-03T05:26:12,726 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/a/5a38781bfe8d4be6bc03995b99cd733c, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T05:26:12,727 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/b/479856e6b97142f9bd00d355919c177e as hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/b/479856e6b97142f9bd00d355919c177e 2024-12-03T05:26:12,733 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/b/479856e6b97142f9bd00d355919c177e, entries=7, sequenceid=26, filesize=5.4 K 2024-12-03T05:26:12,734 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/.tmp/c/8050c870e5134c298c5bde2deab65c92 as hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/c/8050c870e5134c298c5bde2deab65c92 2024-12-03T05:26:12,740 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/c/8050c870e5134c298c5bde2deab65c92, entries=6, sequenceid=26, filesize=5.3 K 2024-12-03T05:26:12,740 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e82ebda03d2152966c59ec19cd109b2d in 119ms, sequenceid=26, compaction requested=false; wal=null 2024-12-03T05:26:12,741 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/0000000000000000026 2024-12-03T05:26:12,742 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,742 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,743 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsAfterAbortingFlush descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:12,744 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,746 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsAfterAbortingFlush/e82ebda03d2152966c59ec19cd109b2d/recovered.edits/26.seqid, newMaxSeqId=26, maxSeqId=1 2024-12-03T05:26:12,747 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e82ebda03d2152966c59ec19cd109b2d; next sequenceid=27; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65475351, jitterRate=-0.024341240525245667}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:12,747 DEBUG [Time-limited test {}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e82ebda03d2152966c59ec19cd109b2d 2024-12-03T05:26:12,748 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e82ebda03d2152966c59ec19cd109b2d: Running coprocessor pre-open hook at 1733203572574Writing region info on filesystem at 1733203572574Initializing all the Stores at 1733203572575 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203572575Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203572576 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203572576Obtaining lock to block concurrent updates at 1733203572621 (+45 ms)Preparing flush snapshotting stores in e82ebda03d2152966c59ec19cd109b2d at 1733203572621Finished memstore snapshotting testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d., syncing WAL and waiting on mvcc, flushsize=dataSize=1190, getHeapSize=3440, getOffHeapSize=0, getCellsCount=20 at 1733203572622 (+1 ms)Flushing stores of testReplayEditsAfterAbortingFlush,,1733203563109.e82ebda03d2152966c59ec19cd109b2d. at 1733203572622Flushing e82ebda03d2152966c59ec19cd109b2d/a: creating writer at 1733203572622Flushing e82ebda03d2152966c59ec19cd109b2d/a: appending metadata at 1733203572635 (+13 ms)Flushing e82ebda03d2152966c59ec19cd109b2d/a: closing flushed file at 1733203572635Flushing e82ebda03d2152966c59ec19cd109b2d/b: creating writer at 1733203572647 (+12 ms)Flushing e82ebda03d2152966c59ec19cd109b2d/b: appending metadata at 1733203572667 (+20 ms)Flushing e82ebda03d2152966c59ec19cd109b2d/b: closing flushed file at 1733203572667Flushing e82ebda03d2152966c59ec19cd109b2d/c: creating writer at 1733203572682 (+15 ms)Flushing e82ebda03d2152966c59ec19cd109b2d/c: appending metadata at 1733203572702 (+20 ms)Flushing e82ebda03d2152966c59ec19cd109b2d/c: closing flushed file at 1733203572702Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b742b44: reopening flushed file at 1733203572719 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1f2544de: reopening flushed file at 1733203572726 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3009f733: reopening flushed file at 1733203572733 (+7 ms)Finished flush of dataSize ~1.16 KB/1190, heapSize ~3.36 KB/3440, currentSize=0 B/0 for e82ebda03d2152966c59ec19cd109b2d in 119ms, sequenceid=26, compaction requested=false; wal=null at 1733203572740 (+7 ms)Cleaning up temporary data from old regions at 1733203572742 (+2 ms)Running coprocessor post-open hooks at 1733203572747 (+5 ms)Region opened successfully at 1733203572748 (+1 ms) 2024-12-03T05:26:12,769 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsAfterAbortingFlush Thread=413 (was 411) Potentially hanging thread: AsyncFSWAL-22-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:53336 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47068 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-22-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57854 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741905_1083] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:53368 [Waiting for operation #17] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57858 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47052 [Waiting for operation #3] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1182 (was 1113) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=689 (was 705), ProcessCount=11 (was 11), AvailableMemoryMB=3516 (was 3130) - AvailableMemoryMB LEAK? - 2024-12-03T05:26:12,769 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T05:26:12,781 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=413, OpenFileDescriptor=1182, MaxFileDescriptor=1048576, SystemLoadAverage=689, ProcessCount=11, AvailableMemoryMB=3515 2024-12-03T05:26:12,781 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1182 is superior to 1024 2024-12-03T05:26:12,801 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:12,803 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:12,804 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:26:12,807 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-43366538, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-43366538, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:12,822 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-43366538/hregion-43366538.1733203572807, exclude list is [], retry=0 2024-12-03T05:26:12,825 DEBUG [AsyncFSWAL-24-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:12,825 DEBUG [AsyncFSWAL-24-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:12,825 DEBUG [AsyncFSWAL-24-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:12,848 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-43366538/hregion-43366538.1733203572807 2024-12-03T05:26:12,848 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:26:12,849 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 33a43bab23af1c35a75fdc227aca90d9, NAME => 'testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testDatalossWhenInputError', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:26:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741910_1088 (size=61) 2024-12-03T05:26:12,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741910_1088 (size=61) 2024-12-03T05:26:12,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741910_1088 (size=61) 2024-12-03T05:26:12,868 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:12,869 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,871 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33a43bab23af1c35a75fdc227aca90d9 columnFamilyName a 2024-12-03T05:26:12,871 DEBUG [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:12,872 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(327): Store=33a43bab23af1c35a75fdc227aca90d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:12,872 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,873 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,873 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,874 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,875 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,877 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:26:12,878 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33a43bab23af1c35a75fdc227aca90d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62123042, jitterRate=-0.07429453730583191}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:26:12,879 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33a43bab23af1c35a75fdc227aca90d9: Writing region info on filesystem at 1733203572868Initializing all the Stores at 1733203572869 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203572869Cleaning up temporary data from old regions at 1733203572874 (+5 ms)Region opened successfully at 1733203572879 (+5 ms) 2024-12-03T05:26:12,879 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33a43bab23af1c35a75fdc227aca90d9, disabling compactions & flushes 2024-12-03T05:26:12,879 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,879 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,879 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. after waiting 0 ms 2024-12-03T05:26:12,879 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,880 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,880 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33a43bab23af1c35a75fdc227aca90d9: Waiting for close lock at 1733203572879Disabling compacts and flushes for region at 1733203572879Disabling writes for close at 1733203572879Writing region close event to WAL at 1733203572879Closed at 1733203572879 2024-12-03T05:26:12,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741909_1087 (size=175) 2024-12-03T05:26:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741909_1087 (size=175) 2024-12-03T05:26:12,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741909_1087 (size=175) 2024-12-03T05:26:12,885 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:12,885 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-43366538:(num 1733203572807) 2024-12-03T05:26:12,885 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:12,893 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:12,912 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, exclude list is [], retry=0 2024-12-03T05:26:12,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:12,915 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:12,916 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:12,948 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 2024-12-03T05:26:12,949 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:26:12,949 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33a43bab23af1c35a75fdc227aca90d9, NAME => 'testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:12,950 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:12,950 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,950 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,952 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,955 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33a43bab23af1c35a75fdc227aca90d9 columnFamilyName a 2024-12-03T05:26:12,956 DEBUG [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:12,956 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(327): Store=33a43bab23af1c35a75fdc227aca90d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:12,957 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,958 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,962 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,962 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,963 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,966 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:12,967 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33a43bab23af1c35a75fdc227aca90d9; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67497456, jitterRate=0.0057904720306396484}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:26:12,968 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33a43bab23af1c35a75fdc227aca90d9: Writing region info on filesystem at 1733203572950Initializing all the Stores at 1733203572951 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203572951Cleaning up temporary data from old regions at 1733203572963 (+12 ms)Region opened successfully at 1733203572968 (+5 ms) 2024-12-03T05:26:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 33a43bab23af1c35a75fdc227aca90d9, disabling compactions & flushes 2024-12-03T05:26:12,978 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. after waiting 0 ms 2024-12-03T05:26:12,978 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,979 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 750 in region testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,979 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. 2024-12-03T05:26:12,979 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 33a43bab23af1c35a75fdc227aca90d9: Waiting for close lock at 1733203572978Disabling compacts and flushes for region at 1733203572978Disabling writes for close at 1733203572978Writing region close event to WAL at 1733203572979 (+1 ms)Closed at 1733203572979 2024-12-03T05:26:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741911_1089 (size=1795) 2024-12-03T05:26:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741911_1089 (size=1795) 2024-12-03T05:26:12,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741911_1089 (size=1795) 2024-12-03T05:26:13,006 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, size=1.8 K (1795bytes) 2024-12-03T05:26:13,006 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 2024-12-03T05:26:13,007 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 after 1ms 2024-12-03T05:26:13,038 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:13,038 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 took 32ms 2024-12-03T05:26:13,040 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 so closing down 2024-12-03T05:26:13,040 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:13,041 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733203572893.temp 2024-12-03T05:26:13,062 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp 2024-12-03T05:26:13,062 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741912_1090 (size=1795) 2024-12-03T05:26:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741912_1090 (size=1795) 2024-12-03T05:26:13,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741912_1090 (size=1795) 2024-12-03T05:26:13,070 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp (wrote 10 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:13,071 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp to hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 2024-12-03T05:26:13,071 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, size=1.8 K, length=1795, corrupted=false, cancelled=false 2024-12-03T05:26:13,071 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, journal: Splitting hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, size=1.8 K (1795bytes) at 1733203573006Finishing writing output for hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 so closing down at 1733203573040 (+34 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp at 1733203573062 (+22 ms)3 split writer threads finished at 1733203573062Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp (wrote 10 edits, skipped 0 edits in 0 ms) at 1733203573070 (+8 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000003-wal.1733203572893.temp to hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 at 1733203573071 (+1 ms)Processed 10 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893, size=1.8 K, length=1795, corrupted=false, cancelled=false at 1733203573071 2024-12-03T05:26:13,073 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203572893 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203572893 2024-12-03T05:26:13,074 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 2024-12-03T05:26:13,101 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:13,552 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:13,554 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:13,573 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203573555, exclude list is [], retry=0 2024-12-03T05:26:13,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:13,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:13,575 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:13,603 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203573555 2024-12-03T05:26:13,603 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:26:13,603 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33a43bab23af1c35a75fdc227aca90d9, NAME => 'testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:13,603 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:13,603 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,603 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,608 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,610 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33a43bab23af1c35a75fdc227aca90d9 columnFamilyName a 2024-12-03T05:26:13,610 DEBUG [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:13,611 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(327): Store=33a43bab23af1c35a75fdc227aca90d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:13,612 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,613 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,614 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,615 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 2024-12-03T05:26:13,642 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:13,643 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 10, skipped 0, firstSequenceIdInLog=3, maxSequenceIdInLog=12, path=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 2024-12-03T05:26:13,643 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 33a43bab23af1c35a75fdc227aca90d9 1/1 column families, dataSize=750 B heapSize=1.73 KB 2024-12-03T05:26:13,664 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/.tmp/a/0085238581b944ceadce297a5f028c04 is 79, key is testDatalossWhenInputError/a:x0/1733203572968/Put/seqid=0 2024-12-03T05:26:13,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741914_1092 (size=5808) 2024-12-03T05:26:13,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741914_1092 (size=5808) 2024-12-03T05:26:13,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741914_1092 (size=5808) 2024-12-03T05:26:13,675 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=750 B at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/.tmp/a/0085238581b944ceadce297a5f028c04 2024-12-03T05:26:13,684 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/.tmp/a/0085238581b944ceadce297a5f028c04 as hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/a/0085238581b944ceadce297a5f028c04 2024-12-03T05:26:13,694 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/a/0085238581b944ceadce297a5f028c04, entries=10, sequenceid=12, filesize=5.7 K 2024-12-03T05:26:13,694 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 33a43bab23af1c35a75fdc227aca90d9 in 51ms, sequenceid=12, compaction requested=false; wal=null 2024-12-03T05:26:13,695 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/0000000000000000012 2024-12-03T05:26:13,696 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,696 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,701 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,705 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-03T05:26:13,707 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33a43bab23af1c35a75fdc227aca90d9; next sequenceid=13; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69961620, jitterRate=0.042509377002716064}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:26:13,708 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33a43bab23af1c35a75fdc227aca90d9: Writing region info on filesystem at 1733203573603Initializing all the Stores at 1733203573606 (+3 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203573606Obtaining lock to block concurrent updates at 1733203573643 (+37 ms)Preparing flush snapshotting stores in 33a43bab23af1c35a75fdc227aca90d9 at 1733203573643Finished memstore snapshotting testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9., syncing WAL and waiting on mvcc, flushsize=dataSize=750, getHeapSize=1760, getOffHeapSize=0, getCellsCount=10 at 1733203573643Flushing stores of testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9. at 1733203573643Flushing 33a43bab23af1c35a75fdc227aca90d9/a: creating writer at 1733203573643Flushing 33a43bab23af1c35a75fdc227aca90d9/a: appending metadata at 1733203573664 (+21 ms)Flushing 33a43bab23af1c35a75fdc227aca90d9/a: closing flushed file at 1733203573664Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@10ef5f0f: reopening flushed file at 1733203573682 (+18 ms)Finished flush of dataSize ~750 B/750, heapSize ~1.72 KB/1760, currentSize=0 B/0 for 33a43bab23af1c35a75fdc227aca90d9 in 51ms, sequenceid=12, compaction requested=false; wal=null at 1733203573694 (+12 ms)Cleaning up temporary data from old regions at 1733203573696 (+2 ms)Region opened successfully at 1733203573707 (+11 ms) 2024-12-03T05:26:13,712 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 33a43bab23af1c35a75fdc227aca90d9, NAME => 'testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:13,712 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testDatalossWhenInputError,,1733203572802.33a43bab23af1c35a75fdc227aca90d9.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:13,712 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,712 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,714 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,715 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33a43bab23af1c35a75fdc227aca90d9 columnFamilyName a 2024-12-03T05:26:13,715 DEBUG [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:13,724 DEBUG [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/a/0085238581b944ceadce297a5f028c04 2024-12-03T05:26:13,724 INFO [StoreOpener-33a43bab23af1c35a75fdc227aca90d9-1 {}] regionserver.HStore(327): Store=33a43bab23af1c35a75fdc227aca90d9/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:13,724 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,725 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,727 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,727 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,727 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,732 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 33a43bab23af1c35a75fdc227aca90d9 2024-12-03T05:26:13,737 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testDatalossWhenInputError/33a43bab23af1c35a75fdc227aca90d9/recovered.edits/13.seqid, newMaxSeqId=13, maxSeqId=12 2024-12-03T05:26:13,739 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 33a43bab23af1c35a75fdc227aca90d9; next sequenceid=14; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66402094, jitterRate=-0.010531693696975708}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-03T05:26:13,739 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 33a43bab23af1c35a75fdc227aca90d9: Writing region info on filesystem at 1733203573712Initializing all the Stores at 1733203573713 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203573713Cleaning up temporary data from old regions at 1733203573727 (+14 ms)Region opened successfully at 1733203573739 (+12 ms) 2024-12-03T05:26:13,761 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testDatalossWhenInputError Thread=423 (was 413) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57934 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:53424 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57858 [Waiting for operation #18] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: PacketResponder: BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47144 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741913_1091] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-24-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47052 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1264 (was 1182) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=689 (was 689), ProcessCount=11 (was 11), AvailableMemoryMB=3406 (was 3515) 2024-12-03T05:26:13,762 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T05:26:13,774 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=423, OpenFileDescriptor=1264, MaxFileDescriptor=1048576, SystemLoadAverage=689, ProcessCount=11, AvailableMemoryMB=3403 2024-12-03T05:26:13,774 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1264 is superior to 1024 2024-12-03T05:26:13,812 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:13,814 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:13,814 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:26:13,817 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-56206960, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-56206960, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:13,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError 2024-12-03T05:26:13,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testDatalossWhenInputError Metrics about Tables on a single HBase RegionServer 2024-12-03T05:26:13,834 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-56206960/hregion-56206960.1733203573817, exclude list is [], retry=0 2024-12-03T05:26:13,840 DEBUG [AsyncFSWAL-26-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:13,841 DEBUG [AsyncFSWAL-26-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:13,841 DEBUG [AsyncFSWAL-26-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:13,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-56206960/hregion-56206960.1733203573817 2024-12-03T05:26:13,869 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667)] 2024-12-03T05:26:13,869 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => e3a32e910db90dff08d0ebc90bfa1972, NAME => 'testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testCompactedBulkLoadedFiles', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:26:13,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741916_1094 (size=63) 2024-12-03T05:26:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741916_1094 (size=63) 2024-12-03T05:26:13,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741916_1094 (size=63) 2024-12-03T05:26:13,888 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:13,889 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,891 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName a 2024-12-03T05:26:13,891 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:13,893 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:13,893 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,895 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName b 2024-12-03T05:26:13,895 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:13,896 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:13,896 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,899 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName c 2024-12-03T05:26:13,899 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:13,905 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:13,905 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,911 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,912 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,920 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,920 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,921 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:13,923 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:13,933 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:26:13,934 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a32e910db90dff08d0ebc90bfa1972; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58847627, jitterRate=-0.12310202419757843}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a32e910db90dff08d0ebc90bfa1972: Writing region info on filesystem at 1733203573888Initializing all the Stores at 1733203573889 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203573889Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203573889Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203573889Cleaning up temporary data from old regions at 1733203573920 (+31 ms)Region opened successfully at 1733203573935 (+15 ms) 2024-12-03T05:26:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3a32e910db90dff08d0ebc90bfa1972, disabling compactions & flushes 2024-12-03T05:26:13,935 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. after waiting 0 ms 2024-12-03T05:26:13,935 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:13,936 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:13,936 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3a32e910db90dff08d0ebc90bfa1972: Waiting for close lock at 1733203573935Disabling compacts and flushes for region at 1733203573935Disabling writes for close at 1733203573935Writing region close event to WAL at 1733203573936 (+1 ms)Closed at 1733203573936 2024-12-03T05:26:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741915_1093 (size=175) 2024-12-03T05:26:13,949 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /hbase/WALs/hregion-56206960/hregion-56206960.1733203573817 not finished, retry = 0 2024-12-03T05:26:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741915_1093 (size=175) 2024-12-03T05:26:13,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741915_1093 (size=175) 2024-12-03T05:26:14,052 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:14,053 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-56206960:(num 1733203573817) 2024-12-03T05:26:14,053 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:14,056 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:14,074 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, exclude list is [], retry=0 2024-12-03T05:26:14,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:14,077 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:14,078 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:14,115 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 2024-12-03T05:26:14,116 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:26:14,116 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e3a32e910db90dff08d0ebc90bfa1972, NAME => 'testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:14,116 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:14,117 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,117 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,118 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,119 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName a 2024-12-03T05:26:14,119 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:14,119 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:14,120 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,120 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName b 2024-12-03T05:26:14,120 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:14,121 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:14,121 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,121 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName c 2024-12-03T05:26:14,122 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:14,122 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:14,122 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,123 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,124 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,125 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,126 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:14,130 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:14,131 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a32e910db90dff08d0ebc90bfa1972; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74087526, jitterRate=0.10399016737937927}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:14,131 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a32e910db90dff08d0ebc90bfa1972: Writing region info on filesystem at 1733203574117Initializing all the Stores at 1733203574118 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203574118Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203574118Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203574118Cleaning up temporary data from old regions at 1733203574125 (+7 ms)Region opened successfully at 1733203574131 (+6 ms) 2024-12-03T05:26:14,140 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile0 is 32, key is 000/a:a/1733203574139/Put/seqid=0 2024-12-03T05:26:14,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741918_1096 (size=4875) 2024-12-03T05:26:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741918_1096 (size=4875) 2024-12-03T05:26:14,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741918_1096 (size=4875) 2024-12-03T05:26:14,163 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile1 is 32, key is 100/a:a/1733203574162/Put/seqid=0 2024-12-03T05:26:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741919_1097 (size=4875) 2024-12-03T05:26:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741919_1097 (size=4875) 2024-12-03T05:26:14,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741919_1097 (size=4875) 2024-12-03T05:26:14,175 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile2 is 32, key is 200/a:a/1733203574174/Put/seqid=0 2024-12-03T05:26:14,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741920_1098 (size=4875) 2024-12-03T05:26:14,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741920_1098 (size=4875) 2024-12-03T05:26:14,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741920_1098 (size=4875) 2024-12-03T05:26:14,208 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile0 for inclusion in e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,217 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=000 last=050 2024-12-03T05:26:14,217 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T05:26:14,217 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile1 for inclusion in e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,228 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=100 last=150 2024-12-03T05:26:14,228 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T05:26:14,228 INFO [Time-limited test {}] regionserver.HStore(614): Validating hfile at hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile2 for inclusion in e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,236 DEBUG [Time-limited test {}] regionserver.HStore(626): HFile bounds: first=200 last=250 2024-12-03T05:26:14,236 DEBUG [Time-limited test {}] regionserver.HStore(628): Region bounds: first= last= 2024-12-03T05:26:14,236 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing e3a32e910db90dff08d0ebc90bfa1972 3/3 column families, dataSize=51 B heapSize=896 B 2024-12-03T05:26:14,262 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp/a/04a05fb8f8eb4805b7234311747cd3c7 is 55, key is testCompactedBulkLoadedFiles/a:a/1733203574132/Put/seqid=0 2024-12-03T05:26:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741921_1099 (size=5107) 2024-12-03T05:26:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741921_1099 (size=5107) 2024-12-03T05:26:14,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741921_1099 (size=5107) 2024-12-03T05:26:14,285 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51 B at sequenceid=4 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp/a/04a05fb8f8eb4805b7234311747cd3c7 2024-12-03T05:26:14,294 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp/a/04a05fb8f8eb4805b7234311747cd3c7 as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7 2024-12-03T05:26:14,319 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7, entries=1, sequenceid=4, filesize=5.0 K 2024-12-03T05:26:14,325 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~51 B/51, heapSize ~368 B/368, currentSize=0 B/0 for e3a32e910db90dff08d0ebc90bfa1972 in 89ms, sequenceid=4, compaction requested=false 2024-12-03T05:26:14,326 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for e3a32e910db90dff08d0ebc90bfa1972: 2024-12-03T05:26:14,328 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile0 as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ 2024-12-03T05:26:14,334 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile1 as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ 2024-12-03T05:26:14,338 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile2 as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ 2024-12-03T05:26:14,340 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile0 into e3a32e910db90dff08d0ebc90bfa1972/a as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ - updating store file list. 2024-12-03T05:26:14,348 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:14,348 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ into e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,348 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile0 into e3a32e910db90dff08d0ebc90bfa1972/a (new location: hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_) 2024-12-03T05:26:14,349 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile1 into e3a32e910db90dff08d0ebc90bfa1972/a as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ - updating store file list. 2024-12-03T05:26:14,354 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:14,354 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ into e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,354 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile1 into e3a32e910db90dff08d0ebc90bfa1972/a (new location: hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_) 2024-12-03T05:26:14,356 INFO [Time-limited test {}] regionserver.HStore(700): Loaded HFile hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile2 into e3a32e910db90dff08d0ebc90bfa1972/a as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ - updating store file list. 2024-12-03T05:26:14,368 DEBUG [Time-limited test {}] regionserver.HStoreFile(483): HFile Bloom filter type for bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:14,369 INFO [Time-limited test {}] regionserver.HStore(722): Loaded HFile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ into e3a32e910db90dff08d0ebc90bfa1972/a 2024-12-03T05:26:14,369 INFO [Time-limited test {}] regionserver.HStore(706): Successfully loaded hdfs://localhost:43427/hbase/testCompactedBulkLoadedFiles/hfile2 into e3a32e910db90dff08d0ebc90bfa1972/a (new location: hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_) 2024-12-03T05:26:14,387 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-12-03T05:26:14,388 DEBUG [Time-limited test {}] regionserver.HStore(1541): e3a32e910db90dff08d0ebc90bfa1972/a is initiating major compaction (all files) 2024-12-03T05:26:14,388 INFO [Time-limited test {}] regionserver.HRegion(2416): Starting compaction of e3a32e910db90dff08d0ebc90bfa1972/a in testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:14,388 INFO [Time-limited test {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_] into tmpdir=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp, totalSize=19.3 K 2024-12-03T05:26:14,389 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 04a05fb8f8eb4805b7234311747cd3c7, keycount=1, bloomtype=ROW, size=5.0 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=1733203574132 2024-12-03T05:26:14,391 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T05:26:14,393 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting 5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T05:26:14,393 DEBUG [Time-limited test {}] compactions.Compactor(225): Compacting bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_, keycount=10, bloomtype=NONE, size=4.8 K, encoding=NONE, compression=NONE, seqNum=4, earliestPutTs=-9223372036854775808 2024-12-03T05:26:14,422 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp/a/8a86e553ca304e2591955f0ee3c805f7 is 55, key is testCompactedBulkLoadedFiles/a:a/1733203574132/Put/seqid=0 2024-12-03T05:26:14,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741922_1100 (size=6154) 2024-12-03T05:26:14,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741922_1100 (size=6154) 2024-12-03T05:26:14,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741922_1100 (size=6154) 2024-12-03T05:26:14,871 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/.tmp/a/8a86e553ca304e2591955f0ee3c805f7 as hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/8a86e553ca304e2591955f0ee3c805f7 2024-12-03T05:26:14,878 INFO [Time-limited test {}] regionserver.HStore(1337): Completed major compaction of 4 (all) file(s) in e3a32e910db90dff08d0ebc90bfa1972/a of e3a32e910db90dff08d0ebc90bfa1972 into 8a86e553ca304e2591955f0ee3c805f7(size=6.0 K), total size for store is 6.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-12-03T05:26:14,879 DEBUG [Time-limited test {}] regionserver.HRegion(2446): Compaction status journal for e3a32e910db90dff08d0ebc90bfa1972: 2024-12-03T05:26:14,879 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T05:26:14,879 DEBUG [Time-limited test {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 0 store files, 0 compacting, 0 eligible, 16 blocking 2024-12-03T05:26:14,909 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, size=0 (0bytes) 2024-12-03T05:26:14,909 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 might be still open, length is 0 2024-12-03T05:26:14,909 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 2024-12-03T05:26:14,910 WARN [IPC Server handler 1 on default port 43427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 has not been closed. Lease recovery is in progress. RecoveryId = 1101 for block blk_1073741917_1095 2024-12-03T05:26:14,910 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 after 1ms 2024-12-03T05:26:17,155 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47182 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45109:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47182 dst: /127.0.0.1:45109 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45109 remote=/127.0.0.1:47182]. Total timeout mills is 60000, 57723 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:17,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:53462 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:40897:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53462 dst: /127.0.0.1:40897 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:17,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:57986 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741917_1095] {}] datanode.DataXceiver(331): 127.0.0.1:45403:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:57986 dst: /127.0.0.1:45403 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:17,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741917_1101 (size=1794) 2024-12-03T05:26:17,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741917_1101 (size=1794) 2024-12-03T05:26:17,160 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741917_1101 (size=1794) 2024-12-03T05:26:17,954 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-03T05:26:18,362 INFO [master/e3c6570475b6:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-03T05:26:18,362 INFO [master/e3c6570475b6:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-03T05:26:18,911 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 after 4002ms 2024-12-03T05:26:18,939 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:18,939 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 took 4030ms 2024-12-03T05:26:18,942 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056; continuing. 2024-12-03T05:26:18,942 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 so closing down 2024-12-03T05:26:18,942 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:18,943 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733203574056.temp 2024-12-03T05:26:18,971 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp 2024-12-03T05:26:18,971 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:18,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741923_1102 (size=800) 2024-12-03T05:26:18,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741923_1102 (size=800) 2024-12-03T05:26:18,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741923_1102 (size=800) 2024-12-03T05:26:18,980 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp (wrote 2 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:18,981 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp to hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 2024-12-03T05:26:18,981 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 5 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T05:26:18,981 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, journal: Splitting hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, size=0 (0bytes) at 1733203574909Finishing writing output for hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 so closing down at 1733203578942 (+4033 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp at 1733203578971 (+29 ms)3 split writer threads finished at 1733203578971Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp (wrote 2 edits, skipped 0 edits in 0 ms) at 1733203578980 (+9 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000003-wal.1733203574056.temp to hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 at 1733203578981 (+1 ms)Processed 5 edits across 1 Regions in 41 ms; skipped=3; WAL=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056, size=0, length=0, corrupted=false, cancelled=false at 1733203578981 2024-12-03T05:26:18,983 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203574056 2024-12-03T05:26:18,984 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 2024-12-03T05:26:18,984 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:18,986 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:18,998 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203578986, exclude list is [], retry=0 2024-12-03T05:26:19,001 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:19,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:19,002 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:19,028 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203578986 2024-12-03T05:26:19,029 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:19,029 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => e3a32e910db90dff08d0ebc90bfa1972, NAME => 'testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:19,029 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:19,029 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,029 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,031 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,032 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName a 2024-12-03T05:26:19,032 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,038 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7 2024-12-03T05:26:19,042 DEBUG [StoreFileOpener-e3a32e910db90dff08d0ebc90bfa1972-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:19,042 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ 2024-12-03T05:26:19,046 DEBUG [StoreFileOpener-e3a32e910db90dff08d0ebc90bfa1972-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for 5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:19,046 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ 2024-12-03T05:26:19,049 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/8a86e553ca304e2591955f0ee3c805f7 2024-12-03T05:26:19,054 DEBUG [StoreFileOpener-e3a32e910db90dff08d0ebc90bfa1972-a-1 {}] regionserver.HStoreFile(483): HFile Bloom filter type for bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_: NONE, but ROW specified in column family configuration 2024-12-03T05:26:19,054 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ 2024-12-03T05:26:19,055 WARN [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7 from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@483d0d1f 2024-12-03T05:26:19,055 WARN [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@483d0d1f 2024-12-03T05:26:19,055 WARN [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@483d0d1f 2024-12-03T05:26:19,055 WARN [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(317): Clearing the compacted storefile hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ from org.apache.hadoop.hbase.regionserver.DefaultStoreEngine@483d0d1f 2024-12-03T05:26:19,055 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.StoreEngine(327): Moving the files [hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_] to archive 2024-12-03T05:26:19,056 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-12-03T05:26:19,057 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7 to hdfs://localhost:43427/hbase/archive/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/04a05fb8f8eb4805b7234311747cd3c7 2024-12-03T05:26:19,059 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ to hdfs://localhost:43427/hbase/archive/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_ 2024-12-03T05:26:19,060 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ to hdfs://localhost:43427/hbase/archive/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_ 2024-12-03T05:26:19,062 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ to hdfs://localhost:43427/hbase/archive/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/a/bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_ 2024-12-03T05:26:19,062 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,062 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,063 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName b 2024-12-03T05:26:19,063 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,064 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,064 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,065 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e3a32e910db90dff08d0ebc90bfa1972 columnFamilyName c 2024-12-03T05:26:19,065 DEBUG [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,065 INFO [StoreOpener-e3a32e910db90dff08d0ebc90bfa1972-1 {}] regionserver.HStore(327): Store=e3a32e910db90dff08d0ebc90bfa1972/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,065 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,066 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,067 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,068 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 2024-12-03T05:26:19,093 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:19,095 DEBUG [Time-limited test {}] regionserver.HRegion(5836): e3a32e910db90dff08d0ebc90bfa1972 : Replaying compaction marker table_name: "testCompactedBulkLoadedFiles" encoded_region_name: "e3a32e910db90dff08d0ebc90bfa1972" family_name: "a" compaction_input: "04a05fb8f8eb4805b7234311747cd3c7" compaction_input: "3ff1e53ab1f64327bbdcf0166aca9508_SeqId_4_" compaction_input: "5acea6ddc0cc477c9049edf08d7b75f5_SeqId_4_" compaction_input: "bca63c7ed85c4c79a5d5c885e3f23e5d_SeqId_4_" compaction_output: "8a86e553ca304e2591955f0ee3c805f7" store_home_dir: "a" region_name: "testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972." with seqId=9223372036854775807 and lastReplayedOpenRegionSeqId=-1 2024-12-03T05:26:19,095 DEBUG [Time-limited test {}] regionserver.HStore(1354): Completing compaction from the WAL marker 2024-12-03T05:26:19,095 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 0, skipped 2, firstSequenceIdInLog=3, maxSequenceIdInLog=8, path=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 2024-12-03T05:26:19,096 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/0000000000000000008 2024-12-03T05:26:19,097 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,097 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,098 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testCompactedBulkLoadedFiles descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:19,100 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for e3a32e910db90dff08d0ebc90bfa1972 2024-12-03T05:26:19,102 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testCompactedBulkLoadedFiles/e3a32e910db90dff08d0ebc90bfa1972/recovered.edits/8.seqid, newMaxSeqId=8, maxSeqId=1 2024-12-03T05:26:19,104 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened e3a32e910db90dff08d0ebc90bfa1972; next sequenceid=9; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62417775, jitterRate=-0.06990267336368561}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:19,104 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for e3a32e910db90dff08d0ebc90bfa1972: Writing region info on filesystem at 1733203579029Initializing all the Stores at 1733203579030 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579030Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579031 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579031Cleaning up temporary data from old regions at 1733203579097 (+66 ms)Region opened successfully at 1733203579104 (+7 ms) 2024-12-03T05:26:19,106 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing e3a32e910db90dff08d0ebc90bfa1972, disabling compactions & flushes 2024-12-03T05:26:19,106 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:19,106 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:19,106 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. after waiting 0 ms 2024-12-03T05:26:19,106 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:19,107 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testCompactedBulkLoadedFiles,,1733203573813.e3a32e910db90dff08d0ebc90bfa1972. 2024-12-03T05:26:19,107 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for e3a32e910db90dff08d0ebc90bfa1972: Waiting for close lock at 1733203579106Disabling compacts and flushes for region at 1733203579106Disabling writes for close at 1733203579106Writing region close event to WAL at 1733203579107 (+1 ms)Closed at 1733203579107 2024-12-03T05:26:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741924_1103 (size=175) 2024-12-03T05:26:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741924_1103 (size=175) 2024-12-03T05:26:19,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741924_1103 (size=175) 2024-12-03T05:26:19,111 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:19,111 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733203578986) 2024-12-03T05:26:19,133 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testCompactedBulkLoadedFiles Thread=432 (was 423) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2132123048_22 at /127.0.0.1:53542 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: LeaseRenewer:jenkinstestCompactedBulkLoadedFiles@localhost:43427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2132123048_22 at /127.0.0.1:47258 [Waiting for operation #7] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-2132123048_22 at /127.0.0.1:58044 [Waiting for operation #5] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43427 from jenkinstestCompactedBulkLoadedFiles java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-26-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-26-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1338 (was 1264) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=666 (was 689), ProcessCount=11 (was 11), AvailableMemoryMB=4018 (was 3403) - AvailableMemoryMB LEAK? - 2024-12-03T05:26:19,134 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1338 is superior to 1024 2024-12-03T05:26:19,149 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=432, OpenFileDescriptor=1338, MaxFileDescriptor=1048576, SystemLoadAverage=666, ProcessCount=11, AvailableMemoryMB=4014 2024-12-03T05:26:19,150 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1338 is superior to 1024 2024-12-03T05:26:19,169 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:19,171 INFO [Time-limited test {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-03T05:26:19,171 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-03T05:26:19,175 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=hregion-23390036, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/hregion-23390036, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:19,193 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/hregion-23390036/hregion-23390036.1733203579176, exclude list is [], retry=0 2024-12-03T05:26:19,197 DEBUG [AsyncFSWAL-28-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:19,200 DEBUG [AsyncFSWAL-28-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:19,204 DEBUG [AsyncFSWAL-28-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:19,247 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/hregion-23390036/hregion-23390036.1733203579176 2024-12-03T05:26:19,248 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:19,248 INFO [Time-limited test {}] regionserver.HRegion(7572): creating {ENCODED => 2df49c4ef4d46ab98995bc16e3a9f624, NAME => 'testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='testReplayEditsWrittenViaHRegion', {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:43427/hbase 2024-12-03T05:26:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741926_1105 (size=67) 2024-12-03T05:26:19,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741926_1105 (size=67) 2024-12-03T05:26:19,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741926_1105 (size=67) 2024-12-03T05:26:19,259 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:19,264 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,266 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName a 2024-12-03T05:26:19,266 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,266 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,267 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,268 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName b 2024-12-03T05:26:19,268 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,269 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,269 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,271 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName c 2024-12-03T05:26:19,271 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,271 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,271 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,272 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,273 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,273 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,274 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:19,275 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,277 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-03T05:26:19,277 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2df49c4ef4d46ab98995bc16e3a9f624; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62716424, jitterRate=-0.0654524564743042}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:19,277 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2df49c4ef4d46ab98995bc16e3a9f624: Writing region info on filesystem at 1733203579259Initializing all the Stores at 1733203579260 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579260Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579264 (+4 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579264Cleaning up temporary data from old regions at 1733203579273 (+9 ms)Region opened successfully at 1733203579277 (+4 ms) 2024-12-03T05:26:19,278 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2df49c4ef4d46ab98995bc16e3a9f624, disabling compactions & flushes 2024-12-03T05:26:19,278 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,278 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,278 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. after waiting 0 ms 2024-12-03T05:26:19,278 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,280 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,280 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2df49c4ef4d46ab98995bc16e3a9f624: Waiting for close lock at 1733203579277Disabling compacts and flushes for region at 1733203579277Disabling writes for close at 1733203579278 (+1 ms)Writing region close event to WAL at 1733203579280 (+2 ms)Closed at 1733203579280 2024-12-03T05:26:19,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741925_1104 (size=175) 2024-12-03T05:26:19,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741925_1104 (size=175) 2024-12-03T05:26:19,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741925_1104 (size=175) 2024-12-03T05:26:19,285 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:19,285 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL hregion-23390036:(num 1733203579176) 2024-12-03T05:26:19,285 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:19,287 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:19,298 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, exclude list is [], retry=0 2024-12-03T05:26:19,301 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:19,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:19,302 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:19,327 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 2024-12-03T05:26:19,328 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:19,328 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2df49c4ef4d46ab98995bc16e3a9f624, NAME => 'testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:19,328 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:19,328 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,328 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,331 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,332 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName a 2024-12-03T05:26:19,332 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,335 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,335 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,336 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName b 2024-12-03T05:26:19,336 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,337 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,337 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,338 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName c 2024-12-03T05:26:19,338 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,338 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,339 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,339 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,341 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,343 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,343 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,343 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:19,348 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,350 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2df49c4ef4d46ab98995bc16e3a9f624; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65433510, jitterRate=-0.024964720010757446}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:19,350 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2df49c4ef4d46ab98995bc16e3a9f624: Writing region info on filesystem at 1733203579328Initializing all the Stores at 1733203579329 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579329Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579331 (+2 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579331Cleaning up temporary data from old regions at 1733203579343 (+12 ms)Region opened successfully at 1733203579350 (+7 ms) 2024-12-03T05:26:19,363 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2df49c4ef4d46ab98995bc16e3a9f624 3/3 column families, dataSize=870 B heapSize=2.31 KB 2024-12-03T05:26:19,383 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/cc27b4063c5e48679318a33d24e5d952 is 91, key is testReplayEditsWrittenViaHRegion/a:x0/1733203579350/Put/seqid=0 2024-12-03T05:26:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741928_1107 (size=5958) 2024-12-03T05:26:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741928_1107 (size=5958) 2024-12-03T05:26:19,392 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741928_1107 (size=5958) 2024-12-03T05:26:19,393 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/cc27b4063c5e48679318a33d24e5d952 2024-12-03T05:26:19,399 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/cc27b4063c5e48679318a33d24e5d952 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/cc27b4063c5e48679318a33d24e5d952 2024-12-03T05:26:19,405 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/cc27b4063c5e48679318a33d24e5d952, entries=10, sequenceid=13, filesize=5.8 K 2024-12-03T05:26:19,406 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~870 B/870, heapSize ~1.80 KB/1840, currentSize=0 B/0 for 2df49c4ef4d46ab98995bc16e3a9f624 in 44ms, sequenceid=13, compaction requested=false 2024-12-03T05:26:19,406 DEBUG [Time-limited test {}] regionserver.HRegion(2603): Flush status journal for 2df49c4ef4d46ab98995bc16e3a9f624: 2024-12-03T05:26:19,423 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2df49c4ef4d46ab98995bc16e3a9f624, disabling compactions & flushes 2024-12-03T05:26:19,423 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,423 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,423 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. after waiting 0 ms 2024-12-03T05:26:19,423 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,424 ERROR [Time-limited test {}] regionserver.HRegion(1960): Memstore data size is 1740 in region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,424 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:19,424 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2df49c4ef4d46ab98995bc16e3a9f624: Waiting for close lock at 1733203579423Disabling compacts and flushes for region at 1733203579423Disabling writes for close at 1733203579423Writing region close event to WAL at 1733203579424 (+1 ms)Closed at 1733203579424 2024-12-03T05:26:19,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741927_1106 (size=6243) 2024-12-03T05:26:19,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741927_1106 (size=6243) 2024-12-03T05:26:19,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741927_1106 (size=6243) 2024-12-03T05:26:19,445 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, size=6.1 K (6243bytes) 2024-12-03T05:26:19,445 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 2024-12-03T05:26:19,446 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 after 1ms 2024-12-03T05:26:19,469 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:19,469 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 took 24ms 2024-12-03T05:26:19,472 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 so closing down 2024-12-03T05:26:19,472 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:19,473 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000003-wal.1733203579287.temp 2024-12-03T05:26:19,495 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp 2024-12-03T05:26:19,495 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741929_1108 (size=5575) 2024-12-03T05:26:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741929_1108 (size=5575) 2024-12-03T05:26:19,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741929_1108 (size=5575) 2024-12-03T05:26:19,502 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:19,504 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 2024-12-03T05:26:19,504 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, size=6.1 K, length=6243, corrupted=false, cancelled=false 2024-12-03T05:26:19,504 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, journal: Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, size=6.1 K (6243bytes) at 1733203579445Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 so closing down at 1733203579472 (+27 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp at 1733203579495 (+23 ms)3 split writer threads finished at 1733203579495Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733203579502 (+7 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000003-wal.1733203579287.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 at 1733203579504 (+2 ms)Processed 32 edits across 1 Regions in 34 ms; skipped=2; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287, size=6.1 K, length=6243, corrupted=false, cancelled=false at 1733203579504 2024-12-03T05:26:19,506 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579287 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203579287 2024-12-03T05:26:19,506 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 2024-12-03T05:26:19,507 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:19,508 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:19,520 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, exclude list is [], retry=0 2024-12-03T05:26:19,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:19,523 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:19,524 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:19,555 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 2024-12-03T05:26:19,556 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237),(127.0.0.1/127.0.0.1:41549:41549)] 2024-12-03T05:26:19,556 DEBUG [Time-limited test {}] regionserver.HRegion(7752): Opening region: {ENCODED => 2df49c4ef4d46ab98995bc16e3a9f624, NAME => 'testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.', STARTKEY => '', ENDKEY => ''} 2024-12-03T05:26:19,556 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:19,556 DEBUG [Time-limited test {}] regionserver.HRegion(7794): checking encryption for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,556 DEBUG [Time-limited test {}] regionserver.HRegion(7797): checking classloading for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,558 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,559 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName a 2024-12-03T05:26:19,559 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,564 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/cc27b4063c5e48679318a33d24e5d952 2024-12-03T05:26:19,564 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,564 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,565 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName b 2024-12-03T05:26:19,565 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,566 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,566 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,567 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName c 2024-12-03T05:26:19,567 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:19,568 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:19,568 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,569 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,570 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,571 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 2024-12-03T05:26:19,597 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:19,598 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 20, skipped 10, firstSequenceIdInLog=3, maxSequenceIdInLog=35, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 2024-12-03T05:26:19,598 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2df49c4ef4d46ab98995bc16e3a9f624 3/3 column families, dataSize=1.70 KB heapSize=3.88 KB 2024-12-03T05:26:19,616 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/03b509922d684ba5933ea43387ac1a3a is 91, key is testReplayEditsWrittenViaHRegion/b:x0/1733203579406/Put/seqid=0 2024-12-03T05:26:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741931_1110 (size=5958) 2024-12-03T05:26:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741931_1110 (size=5958) 2024-12-03T05:26:19,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741931_1110 (size=5958) 2024-12-03T05:26:19,625 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/03b509922d684ba5933ea43387ac1a3a 2024-12-03T05:26:19,644 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/d858d571daf04ce9aa35fe641c5cdd05 is 91, key is testReplayEditsWrittenViaHRegion/c:x0/1733203579414/Put/seqid=0 2024-12-03T05:26:19,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741932_1111 (size=5958) 2024-12-03T05:26:19,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741932_1111 (size=5958) 2024-12-03T05:26:19,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741932_1111 (size=5958) 2024-12-03T05:26:19,655 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=35 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/d858d571daf04ce9aa35fe641c5cdd05 2024-12-03T05:26:19,662 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/03b509922d684ba5933ea43387ac1a3a as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/b/03b509922d684ba5933ea43387ac1a3a 2024-12-03T05:26:19,668 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/b/03b509922d684ba5933ea43387ac1a3a, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T05:26:19,669 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/d858d571daf04ce9aa35fe641c5cdd05 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/c/d858d571daf04ce9aa35fe641c5cdd05 2024-12-03T05:26:19,676 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/c/d858d571daf04ce9aa35fe641c5cdd05, entries=10, sequenceid=35, filesize=5.8 K 2024-12-03T05:26:19,676 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2df49c4ef4d46ab98995bc16e3a9f624 in 78ms, sequenceid=35, compaction requested=false; wal=null 2024-12-03T05:26:19,677 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000035 2024-12-03T05:26:19,679 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,679 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,680 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:19,681 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:19,684 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/35.seqid, newMaxSeqId=35, maxSeqId=1 2024-12-03T05:26:19,685 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2df49c4ef4d46ab98995bc16e3a9f624; next sequenceid=36; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67709274, jitterRate=0.008946806192398071}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:19,686 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2df49c4ef4d46ab98995bc16e3a9f624: Writing region info on filesystem at 1733203579556Initializing all the Stores at 1733203579557 (+1 ms)Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579557Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579557Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203579557Obtaining lock to block concurrent updates at 1733203579598 (+41 ms)Preparing flush snapshotting stores in 2df49c4ef4d46ab98995bc16e3a9f624 at 1733203579598Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624., syncing WAL and waiting on mvcc, flushsize=dataSize=1740, getHeapSize=3920, getOffHeapSize=0, getCellsCount=20 at 1733203579598Flushing stores of testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. at 1733203579598Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: creating writer at 1733203579598Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: appending metadata at 1733203579615 (+17 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: closing flushed file at 1733203579615Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: creating writer at 1733203579630 (+15 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: appending metadata at 1733203579644 (+14 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: closing flushed file at 1733203579644Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@122e33d9: reopening flushed file at 1733203579661 (+17 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2d81f91c: reopening flushed file at 1733203579668 (+7 ms)Finished flush of dataSize ~1.70 KB/1740, heapSize ~3.59 KB/3680, currentSize=0 B/0 for 2df49c4ef4d46ab98995bc16e3a9f624 in 78ms, sequenceid=35, compaction requested=false; wal=null at 1733203579676 (+8 ms)Cleaning up temporary data from old regions at 1733203579679 (+3 ms)Region opened successfully at 1733203579686 (+7 ms) 2024-12-03T05:26:19,759 INFO [Time-limited test {}] wal.WALSplitter(299): Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, size=0 (0bytes) 2024-12-03T05:26:19,760 WARN [Time-limited test {}] wal.WALSplitter(453): File hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 might be still open, length is 0 2024-12-03T05:26:19,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(134): Recover lease on dfs file hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 2024-12-03T05:26:19,760 WARN [IPC Server handler 3 on default port 43427 {}] namenode.FSNamesystem(3854): DIR* NameSystem.internalReleaseLease: File /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 has not been closed. Lease recovery is in progress. RecoveryId = 1112 for block blk_1073741930_1109 2024-12-03T05:26:19,760 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Failed to recover lease, attempt=0 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 after 0ms 2024-12-03T05:26:20,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:58090 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45403:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:58090 dst: /127.0.0.1:45403 java.io.InterruptedIOException: Interrupted while waiting for IO on channel java.nio.channels.SocketChannel[connected local=localhost/127.0.0.1:45403 remote=/127.0.0.1:58090]. Total timeout mills is 60000, 59564 millis timeout left. at org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:350) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) ~[hadoop-common-3.4.1.jar:?] at java.io.BufferedInputStream.fill(BufferedInputStream.java:244) ~[?:?] at java.io.BufferedInputStream.read1(BufferedInputStream.java:284) ~[?:?] at java.io.BufferedInputStream.read(BufferedInputStream.java:343) ~[?:?] at java.io.DataInputStream.read(DataInputStream.java:151) ~[?:?] at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:20,156 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:47334 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:45109:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:47334 dst: /127.0.0.1:45109 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:20,157 ERROR [DataXceiver for client DFSClient_NONMAPREDUCE_-1397850444_22 at /127.0.0.1:53600 [Receiving block BP-1175346662-172.17.0.3-1733203526961:blk_1073741930_1109] {}] datanode.DataXceiver(331): 127.0.0.1:40897:DataXceiver error processing WRITE_BLOCK operation src: /127.0.0.1:53600 dst: /127.0.0.1:40897 java.io.IOException: Premature EOF from inputStream at org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:216) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) ~[hadoop-hdfs-3.4.1.jar:?] at org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) ~[hadoop-hdfs-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:20,158 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741930_1112 (size=5567) 2024-12-03T05:26:20,159 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741930_1112 (size=5567) 2024-12-03T05:26:23,761 INFO [Time-limited test {}] util.RecoverLeaseFSUtils(223): Recovered lease, attempt=1 on file=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 after 4001ms 2024-12-03T05:26:23,792 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:23,792 INFO [Time-limited test {}] wal.WALSplitter(310): Open hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 took 4033ms 2024-12-03T05:26:23,794 INFO [Time-limited test {}] wal.WALSplitter(493): EOF from hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508; continuing. 2024-12-03T05:26:23,794 DEBUG [Time-limited test {}] wal.WALSplitter(406): Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 so closing down 2024-12-03T05:26:23,794 DEBUG [Time-limited test {}] wal.OutputSink(125): Waiting for split writer threads to finish 2024-12-03T05:26:23,796 INFO [Time-limited test-Writer-0 {}] monitor.StreamSlowMonitor(122): New stream slow monitor 0000000000000000037-wal.1733203579508.temp 2024-12-03T05:26:23,818 INFO [Time-limited test-Writer-0 {}] wal.AbstractRecoveredEditsOutputSink(71): Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp 2024-12-03T05:26:23,818 INFO [Time-limited test {}] wal.OutputSink(145): 3 split writer threads finished 2024-12-03T05:26:23,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741933_1113 (size=5575) 2024-12-03T05:26:23,823 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741933_1113 (size=5575) 2024-12-03T05:26:23,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741933_1113 (size=5575) 2024-12-03T05:26:23,824 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(90): Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp (wrote 30 edits, skipped 0 edits in 0 ms) 2024-12-03T05:26:23,825 INFO [split-log-closeStream-pool-0 {}] wal.AbstractRecoveredEditsOutputSink(123): Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 2024-12-03T05:26:23,826 INFO [Time-limited test {}] wal.WALSplitter(425): Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, size=0, length=0, corrupted=false, cancelled=false 2024-12-03T05:26:23,826 DEBUG [Time-limited test {}] wal.WALSplitter(428): Completed split of hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, journal: Splitting hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, size=0 (0bytes) at 1733203579760Finishing writing output for hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 so closing down at 1733203583794 (+4034 ms)Creating recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp at 1733203583818 (+24 ms)3 split writer threads finished at 1733203583818Closed recovered edits writer path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp (wrote 30 edits, skipped 0 edits in 0 ms) at 1733203583824 (+6 ms)Rename recovered edits hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000037-wal.1733203579508.temp to hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 at 1733203583825 (+1 ms)Processed 30 edits across 1 Regions in 33 ms; skipped=0; WAL=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508, size=0, length=0, corrupted=false, cancelled=false at 1733203583826 (+1 ms) 2024-12-03T05:26:23,827 INFO [Time-limited test {}] wal.WALSplitUtil(143): Moved hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 to hdfs://localhost:43427/hbase/oldWALs/wal.1733203579508 2024-12-03T05:26:23,828 INFO [Time-limited test {}] wal.AbstractTestWALReplay(1167): Split file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 2024-12-03T05:26:23,828 INFO [Time-limited test {}] monitor.StreamSlowMonitor(122): New stream slow monitor monitor 2024-12-03T05:26:23,830 INFO [Time-limited test {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=wal, suffix=, logDir=hdfs://localhost:43427/hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168, archiveDir=hdfs://localhost:43427/hbase/oldWALs, maxLogs=32 2024-12-03T05:26:23,830 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles 2024-12-03T05:26:23,830 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_testCompactedBulkLoadedFiles Metrics about Tables on a single HBase RegionServer 2024-12-03T05:26:23,842 DEBUG [Time-limited test {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203583830, exclude list is [], retry=0 2024-12-03T05:26:23,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45109,DS-f92c31f8-5241-4971-b7df-9d8f22bab352,DISK] 2024-12-03T05:26:23,844 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:45403,DS-9e993d4a-0693-49fe-9591-cfc04dc01190,DISK] 2024-12-03T05:26:23,845 DEBUG [TestAsyncWALReplay-pool-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = /127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:40897,DS-b69d2376-bddb-49fb-bfa0-b05f214e48aa,DISK] 2024-12-03T05:26:23,868 INFO [Time-limited test {}] wal.AbstractFSWAL(991): New WAL /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203583830 2024-12-03T05:26:23,868 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41549:41549),(127.0.0.1/127.0.0.1:42667:42667),(127.0.0.1/127.0.0.1:36237:36237)] 2024-12-03T05:26:23,868 DEBUG [Time-limited test {}] regionserver.HRegion(898): Instantiated testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-03T05:26:23,870 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family a of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,871 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName a 2024-12-03T05:26:23,871 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:23,876 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/cc27b4063c5e48679318a33d24e5d952 2024-12-03T05:26:23,876 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/a, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:23,876 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family b of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,877 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName b 2024-12-03T05:26:23,877 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:23,882 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/b/03b509922d684ba5933ea43387ac1a3a 2024-12-03T05:26:23,882 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/b, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:23,882 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family c of region 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,883 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2df49c4ef4d46ab98995bc16e3a9f624 columnFamilyName c 2024-12-03T05:26:23,883 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-03T05:26:23,888 DEBUG [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.StoreEngine(278): loaded hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/c/d858d571daf04ce9aa35fe641c5cdd05 2024-12-03T05:26:23,888 INFO [StoreOpener-2df49c4ef4d46ab98995bc16e3a9f624-1 {}] regionserver.HStore(327): Store=2df49c4ef4d46ab98995bc16e3a9f624/c, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-03T05:26:23,888 DEBUG [Time-limited test {}] regionserver.HRegion(1038): replaying wal for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,889 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,890 DEBUG [Time-limited test {}] regionserver.HRegion(5546): Found 1 recovered edits file(s) under hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:23,891 INFO [Time-limited test {}] regionserver.HRegion(5613): Replaying edits from hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 2024-12-03T05:26:23,915 DEBUG [Time-limited test {}] wal.AbstractProtobufWALReader(302): Initialized secure protobuf WAL: cipher=AES 2024-12-03T05:26:23,918 DEBUG [Time-limited test {}] regionserver.HRegion(5793): Applied 30, skipped 0, firstSequenceIdInLog=37, maxSequenceIdInLog=66, path=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 2024-12-03T05:26:23,918 INFO [Time-limited test {}] regionserver.HRegion(2902): Flushing 2df49c4ef4d46ab98995bc16e3a9f624 3/3 column families, dataSize=2.55 KB heapSize=5.44 KB 2024-12-03T05:26:23,931 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/decc463612ad4887bc13b4867482c9f8 is 91, key is testReplayEditsWrittenViaHRegion/a:y0/1733203579696/Put/seqid=0 2024-12-03T05:26:23,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741935_1115 (size=5958) 2024-12-03T05:26:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741935_1115 (size=5958) 2024-12-03T05:26:23,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741935_1115 (size=5958) 2024-12-03T05:26:23,939 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/decc463612ad4887bc13b4867482c9f8 2024-12-03T05:26:23,958 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/907c471710f345c183813ac2823708e4 is 91, key is testReplayEditsWrittenViaHRegion/b:y0/1733203579707/Put/seqid=0 2024-12-03T05:26:23,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741936_1116 (size=5958) 2024-12-03T05:26:23,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741936_1116 (size=5958) 2024-12-03T05:26:23,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741936_1116 (size=5958) 2024-12-03T05:26:23,966 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/907c471710f345c183813ac2823708e4 2024-12-03T05:26:23,982 DEBUG [Time-limited test {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/967a9e963f3440f98c5ab32de296be6e is 91, key is testReplayEditsWrittenViaHRegion/c:y0/1733203579714/Put/seqid=0 2024-12-03T05:26:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741937_1117 (size=5958) 2024-12-03T05:26:23,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741937_1117 (size=5958) 2024-12-03T05:26:23,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741937_1117 (size=5958) 2024-12-03T05:26:23,990 INFO [Time-limited test {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=870 B at sequenceid=66 (bloomFilter=true), to=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/967a9e963f3440f98c5ab32de296be6e 2024-12-03T05:26:23,995 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/a/decc463612ad4887bc13b4867482c9f8 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/decc463612ad4887bc13b4867482c9f8 2024-12-03T05:26:23,999 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/a/decc463612ad4887bc13b4867482c9f8, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T05:26:24,000 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/b/907c471710f345c183813ac2823708e4 as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/b/907c471710f345c183813ac2823708e4 2024-12-03T05:26:24,005 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/b/907c471710f345c183813ac2823708e4, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T05:26:24,006 DEBUG [Time-limited test {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/.tmp/c/967a9e963f3440f98c5ab32de296be6e as hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/c/967a9e963f3440f98c5ab32de296be6e 2024-12-03T05:26:24,010 INFO [Time-limited test {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/c/967a9e963f3440f98c5ab32de296be6e, entries=10, sequenceid=66, filesize=5.8 K 2024-12-03T05:26:24,011 INFO [Time-limited test {}] regionserver.HRegion(3140): Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2df49c4ef4d46ab98995bc16e3a9f624 in 93ms, sequenceid=66, compaction requested=false; wal=null 2024-12-03T05:26:24,011 DEBUG [Time-limited test {}] regionserver.HRegion(5420): Deleted recovered.edits file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/0000000000000000066 2024-12-03T05:26:24,012 DEBUG [Time-limited test {}] regionserver.HRegion(1048): stopping wal replay for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:24,012 DEBUG [Time-limited test {}] regionserver.HRegion(1060): Cleaning up temporary data for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:24,013 DEBUG [Time-limited test {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table testReplayEditsWrittenViaHRegion descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-03T05:26:24,014 DEBUG [Time-limited test {}] regionserver.HRegion(1093): writing seq id for 2df49c4ef4d46ab98995bc16e3a9f624 2024-12-03T05:26:24,016 DEBUG [Time-limited test {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/hbase/data/default/testReplayEditsWrittenViaHRegion/2df49c4ef4d46ab98995bc16e3a9f624/recovered.edits/66.seqid, newMaxSeqId=66, maxSeqId=35 2024-12-03T05:26:24,017 INFO [Time-limited test {}] regionserver.HRegion(1114): Opened 2df49c4ef4d46ab98995bc16e3a9f624; next sequenceid=67; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69839042, jitterRate=0.040682822465896606}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-03T05:26:24,017 DEBUG [Time-limited test {}] regionserver.HRegion(1006): Region open journal for 2df49c4ef4d46ab98995bc16e3a9f624: Writing region info on filesystem at 1733203583869Initializing all the Stores at 1733203583869Instantiating store for column family {NAME => 'a', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203583869Instantiating store for column family {NAME => 'b', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203583870 (+1 ms)Instantiating store for column family {NAME => 'c', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733203583870Obtaining lock to block concurrent updates at 1733203583919 (+49 ms)Preparing flush snapshotting stores in 2df49c4ef4d46ab98995bc16e3a9f624 at 1733203583919Finished memstore snapshotting testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624., syncing WAL and waiting on mvcc, flushsize=dataSize=2610, getHeapSize=5520, getOffHeapSize=0, getCellsCount=30 at 1733203583919Flushing stores of testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. at 1733203583919Flushing 2df49c4ef4d46ab98995bc16e3a9f624/a: creating writer at 1733203583919Flushing 2df49c4ef4d46ab98995bc16e3a9f624/a: appending metadata at 1733203583931 (+12 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/a: closing flushed file at 1733203583931Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: creating writer at 1733203583944 (+13 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: appending metadata at 1733203583957 (+13 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/b: closing flushed file at 1733203583957Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: creating writer at 1733203583970 (+13 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: appending metadata at 1733203583982 (+12 ms)Flushing 2df49c4ef4d46ab98995bc16e3a9f624/c: closing flushed file at 1733203583982Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@61e537f1: reopening flushed file at 1733203583994 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1a7d8884: reopening flushed file at 1733203583999 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@744c1672: reopening flushed file at 1733203584005 (+6 ms)Finished flush of dataSize ~2.55 KB/2610, heapSize ~5.39 KB/5520, currentSize=0 B/0 for 2df49c4ef4d46ab98995bc16e3a9f624 in 93ms, sequenceid=66, compaction requested=false; wal=null at 1733203584011 (+6 ms)Cleaning up temporary data from old regions at 1733203584012 (+1 ms)Region opened successfully at 1733203584017 (+5 ms) 2024-12-03T05:26:24,030 DEBUG [Time-limited test {}] regionserver.HRegion(1722): Closing 2df49c4ef4d46ab98995bc16e3a9f624, disabling compactions & flushes 2024-12-03T05:26:24,030 INFO [Time-limited test {}] regionserver.HRegion(1755): Closing region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:24,030 DEBUG [Time-limited test {}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:24,030 DEBUG [Time-limited test {}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. after waiting 0 ms 2024-12-03T05:26:24,030 DEBUG [Time-limited test {}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:24,031 INFO [Time-limited test {}] regionserver.HRegion(1973): Closed testReplayEditsWrittenViaHRegion,,1733203579169.2df49c4ef4d46ab98995bc16e3a9f624. 2024-12-03T05:26:24,031 DEBUG [Time-limited test {}] regionserver.HRegion(1676): Region close journal for 2df49c4ef4d46ab98995bc16e3a9f624: Waiting for close lock at 1733203584029Disabling compacts and flushes for region at 1733203584029Disabling writes for close at 1733203584030 (+1 ms)Writing region close event to WAL at 1733203584031 (+1 ms)Closed at 1733203584031 2024-12-03T05:26:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741934_1114 (size=175) 2024-12-03T05:26:24,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741934_1114 (size=175) 2024-12-03T05:26:24,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741934_1114 (size=175) 2024-12-03T05:26:24,035 DEBUG [Time-limited test {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /hbase/oldWALs 2024-12-03T05:26:24,035 INFO [Time-limited test {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL wal:(num 1733203583830) 2024-12-03T05:26:24,048 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: regionserver.wal.TestSecureAsyncWALReplay#testReplayEditsWrittenViaHRegion Thread=439 (was 432) Potentially hanging thread: LeaseRenewer:jenkinstestReplayEditsWrittenViaHRegion@localhost:43427 java.base@17.0.11/java.lang.Thread.sleep(Native Method) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:441) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) app//org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: IPC Client (1616464420) connection to localhost/127.0.0.1:43427 from jenkinstestReplayEditsWrittenViaHRegion java.base@17.0.11/java.lang.Object.wait(Native Method) app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) Potentially hanging thread: AsyncFSWAL-28-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-3 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1859945236_22 at /127.0.0.1:39148 [Waiting for operation #12] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1859945236_22 at /127.0.0.1:42636 [Waiting for operation #20] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: AsyncFSWAL-28-1 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1859945236_22 at /127.0.0.1:45504 [Waiting for operation #11] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=1404 (was 1338) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=629 (was 666), ProcessCount=11 (was 11), AvailableMemoryMB=3602 (was 4014) 2024-12-03T05:26:24,048 WARN [Time-limited test {}] hbase.ResourceChecker(130): OpenFileDescriptor=1404 is superior to 1024 2024-12-03T05:26:24,048 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-03T05:26:24,048 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:26:24,048 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:26:24,048 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,049 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,049 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-03T05:26:24,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-03T05:26:24,049 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1885054661, stopped=false 2024-12-03T05:26:24,050 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=e3c6570475b6,37365,1733203533327 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:26:24,099 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:26:24,099 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T05:26:24,100 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:26:24,100 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-03T05:26:24,100 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:26:24,100 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,100 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e3c6570475b6,35421,1733203534454' ***** 2024-12-03T05:26:24,100 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T05:26:24,101 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:26:24,101 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'e3c6570475b6,35935,1733203534598' ***** 2024-12-03T05:26:24,101 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T05:26:24,101 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T05:26:24,101 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T05:26:24,101 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T05:26:24,101 INFO [RS:1;e3c6570475b6:35935 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-03T05:26:24,101 INFO [RS:1;e3c6570475b6:35935 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(3091): Received CLOSE for 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:24,101 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(959): stopping server e3c6570475b6,35935,1733203534598 2024-12-03T05:26:24,101 INFO [RS:1;e3c6570475b6:35935 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(959): stopping server e3c6570475b6,35421,1733203534454 2024-12-03T05:26:24,101 INFO [RS:0;e3c6570475b6:35421 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:26:24,102 INFO [RS:1;e3c6570475b6:35935 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:1;e3c6570475b6:35935. 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;e3c6570475b6:35421. 2024-12-03T05:26:24,102 DEBUG [RS:1;e3c6570475b6:35935 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:26:24,102 DEBUG [RS:1;e3c6570475b6:35935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,102 DEBUG [RS:0;e3c6570475b6:35421 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-03T05:26:24,102 DEBUG [RS:0;e3c6570475b6:35421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,102 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(976): stopping server e3c6570475b6,35935,1733203534598; all regions closed. 2024-12-03T05:26:24,102 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3452b8a301e7ba272dae5f6efd7fe4dd, disabling compactions & flushes 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T05:26:24,102 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T05:26:24,102 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:24,102 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. after waiting 0 ms 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-03T05:26:24,102 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:24,102 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-03T05:26:24,102 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-03T05:26:24,103 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1325): Online Regions={3452b8a301e7ba272dae5f6efd7fe4dd=testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd., 1588230740=hbase:meta,,1.1588230740} 2024-12-03T05:26:24,103 DEBUG [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3452b8a301e7ba272dae5f6efd7fe4dd 2024-12-03T05:26:24,103 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-03T05:26:24,103 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-03T05:26:24,103 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-03T05:26:24,103 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-03T05:26:24,103 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-03T05:26:24,103 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=6.86 KB heapSize=11.45 KB 2024-12-03T05:26:24,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741833_1009 (size=175) 2024-12-03T05:26:24,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741833_1009 (size=175) 2024-12-03T05:26:24,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741833_1009 (size=175) 2024-12-03T05:26:24,111 DEBUG [RS:1;e3c6570475b6:35935 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs 2024-12-03T05:26:24,111 INFO [RS:1;e3c6570475b6:35935 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3c6570475b6%2C35935%2C1733203534598:(num 1733203536944) 2024-12-03T05:26:24,111 DEBUG [RS:1;e3c6570475b6:35935 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,111 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:24,111 INFO [RS:1;e3c6570475b6:35935 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] hbase.ChoreService(370): Chore service for: regionserver/e3c6570475b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-03T05:26:24,112 INFO [regionserver/e3c6570475b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:26:24,112 INFO [RS:1;e3c6570475b6:35935 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35935 2024-12-03T05:26:24,117 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/default/testReplayEditsAfterRegionMovedWithMultiCF/3452b8a301e7ba272dae5f6efd7fe4dd/recovered.edits/20.seqid, newMaxSeqId=20, maxSeqId=17 2024-12-03T05:26:24,118 INFO [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:24,118 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3452b8a301e7ba272dae5f6efd7fe4dd: Waiting for close lock at 1733203584102Running coprocessor pre-close hooks at 1733203584102Disabling compacts and flushes for region at 1733203584102Disabling writes for close at 1733203584102Writing region close event to WAL at 1733203584103 (+1 ms)Running coprocessor post-close hooks at 1733203584118 (+15 ms)Closed at 1733203584118 2024-12-03T05:26:24,118 DEBUG [RS_CLOSE_REGION-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd. 2024-12-03T05:26:24,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:26:24,123 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e3c6570475b6,35935,1733203534598 2024-12-03T05:26:24,123 INFO [RS:1;e3c6570475b6:35935 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:26:24,124 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e3c6570475b6,35935,1733203534598] 2024-12-03T05:26:24,136 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/info/1d7b588762504a6f8919c87c1e5d8e85 is 205, key is testReplayEditsAfterRegionMovedWithMultiCF,,1733203558179.3452b8a301e7ba272dae5f6efd7fe4dd./info:regioninfo/1733203562166/Put/seqid=0 2024-12-03T05:26:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741938_1118 (size=8243) 2024-12-03T05:26:24,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741938_1118 (size=8243) 2024-12-03T05:26:24,145 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e3c6570475b6,35935,1733203534598 already deleted, retry=false 2024-12-03T05:26:24,145 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e3c6570475b6,35935,1733203534598 expired; onlineServers=1 2024-12-03T05:26:24,146 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.65 KB at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/info/1d7b588762504a6f8919c87c1e5d8e85 2024-12-03T05:26:24,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741938_1118 (size=8243) 2024-12-03T05:26:24,160 INFO [regionserver/e3c6570475b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:24,163 INFO [regionserver/e3c6570475b6:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:24,169 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/ns/8d591cb039e740a68cc5d1e55c125dac is 43, key is default/ns:d/1733203538229/Put/seqid=0 2024-12-03T05:26:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741939_1119 (size=5153) 2024-12-03T05:26:24,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741939_1119 (size=5153) 2024-12-03T05:26:24,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741939_1119 (size=5153) 2024-12-03T05:26:24,176 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/ns/8d591cb039e740a68cc5d1e55c125dac 2024-12-03T05:26:24,194 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/table/68645247e58148b29165eb7394499680 is 78, key is testReplayEditsAfterRegionMovedWithMultiCF/table:state/1733203559245/Put/seqid=0 2024-12-03T05:26:24,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741940_1120 (size=5431) 2024-12-03T05:26:24,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741940_1120 (size=5431) 2024-12-03T05:26:24,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741940_1120 (size=5431) 2024-12-03T05:26:24,200 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=148 B at sequenceid=21 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/table/68645247e58148b29165eb7394499680 2024-12-03T05:26:24,206 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/info/1d7b588762504a6f8919c87c1e5d8e85 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/info/1d7b588762504a6f8919c87c1e5d8e85 2024-12-03T05:26:24,213 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/info/1d7b588762504a6f8919c87c1e5d8e85, entries=18, sequenceid=21, filesize=8.0 K 2024-12-03T05:26:24,214 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/ns/8d591cb039e740a68cc5d1e55c125dac as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/ns/8d591cb039e740a68cc5d1e55c125dac 2024-12-03T05:26:24,219 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/ns/8d591cb039e740a68cc5d1e55c125dac, entries=2, sequenceid=21, filesize=5.0 K 2024-12-03T05:26:24,220 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/.tmp/table/68645247e58148b29165eb7394499680 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/table/68645247e58148b29165eb7394499680 2024-12-03T05:26:24,226 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/table/68645247e58148b29165eb7394499680, entries=2, sequenceid=21, filesize=5.3 K 2024-12-03T05:26:24,227 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=21, compaction requested=false 2024-12-03T05:26:24,232 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/data/hbase/meta/1588230740/recovered.edits/24.seqid, newMaxSeqId=24, maxSeqId=1 2024-12-03T05:26:24,233 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-03T05:26:24,233 INFO [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-03T05:26:24,233 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733203584103Running coprocessor pre-close hooks at 1733203584103Disabling compacts and flushes for region at 1733203584103Disabling writes for close at 1733203584103Obtaining lock to block concurrent updates at 1733203584103Preparing flush snapshotting stores in 1588230740 at 1733203584103Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=7029, getHeapSize=11664, getOffHeapSize=0, getCellsCount=48 at 1733203584103Flushing stores of hbase:meta,,1.1588230740 at 1733203584105 (+2 ms)Flushing 1588230740/info: creating writer at 1733203584105Flushing 1588230740/info: appending metadata at 1733203584135 (+30 ms)Flushing 1588230740/info: closing flushed file at 1733203584135Flushing 1588230740/ns: creating writer at 1733203584153 (+18 ms)Flushing 1588230740/ns: appending metadata at 1733203584169 (+16 ms)Flushing 1588230740/ns: closing flushed file at 1733203584169Flushing 1588230740/table: creating writer at 1733203584181 (+12 ms)Flushing 1588230740/table: appending metadata at 1733203584194 (+13 ms)Flushing 1588230740/table: closing flushed file at 1733203584194Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2f91c7ce: reopening flushed file at 1733203584206 (+12 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@22012694: reopening flushed file at 1733203584213 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63ce9145: reopening flushed file at 1733203584219 (+6 ms)Finished flush of dataSize ~6.86 KB/7029, heapSize ~11.16 KB/11424, currentSize=0 B/0 for 1588230740 in 124ms, sequenceid=21, compaction requested=false at 1733203584227 (+8 ms)Writing region close event to WAL at 1733203584228 (+1 ms)Running coprocessor post-close hooks at 1733203584233 (+5 ms)Closed at 1733203584233 2024-12-03T05:26:24,233 DEBUG [RS_CLOSE_META-regionserver/e3c6570475b6:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-03T05:26:24,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,234 INFO [RS:1;e3c6570475b6:35935 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:26:24,234 INFO [RS:1;e3c6570475b6:35935 {}] regionserver.HRegionServer(1031): Exiting; stopping=e3c6570475b6,35935,1733203534598; zookeeper connection closed. 2024-12-03T05:26:24,234 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35935-0x10199a715d70002, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,234 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@60e09be9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@60e09be9 2024-12-03T05:26:24,303 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(976): stopping server e3c6570475b6,35421,1733203534454; all regions closed. 2024-12-03T05:26:24,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741836_1012 (size=9487) 2024-12-03T05:26:24,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741836_1012 (size=9487) 2024-12-03T05:26:24,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741836_1012 (size=9487) 2024-12-03T05:26:24,314 DEBUG [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs 2024-12-03T05:26:24,314 INFO [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3c6570475b6%2C35421%2C1733203534454.meta:.meta(num 1733203537722) 2024-12-03T05:26:24,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741835_1011 (size=3621) 2024-12-03T05:26:24,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741835_1011 (size=3621) 2024-12-03T05:26:24,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741835_1011 (size=3621) 2024-12-03T05:26:24,319 DEBUG [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/oldWALs 2024-12-03T05:26:24,319 INFO [RS:0;e3c6570475b6:35421 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL e3c6570475b6%2C35421%2C1733203534454:(num 1733203536945) 2024-12-03T05:26:24,319 DEBUG [RS:0;e3c6570475b6:35421 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-03T05:26:24,319 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.LeaseManager(133): Closed leases 2024-12-03T05:26:24,319 INFO [RS:0;e3c6570475b6:35421 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:26:24,319 INFO [RS:0;e3c6570475b6:35421 {}] hbase.ChoreService(370): Chore service for: regionserver/e3c6570475b6:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-12-03T05:26:24,320 INFO [RS:0;e3c6570475b6:35421 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:26:24,320 INFO [regionserver/e3c6570475b6:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:26:24,320 INFO [RS:0;e3c6570475b6:35421 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:35421 2024-12-03T05:26:24,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-03T05:26:24,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e3c6570475b6,35421,1733203534454 2024-12-03T05:26:24,330 INFO [RS:0;e3c6570475b6:35421 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:26:24,341 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e3c6570475b6,35421,1733203534454] 2024-12-03T05:26:24,351 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/e3c6570475b6,35421,1733203534454 already deleted, retry=false 2024-12-03T05:26:24,351 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; e3c6570475b6,35421,1733203534454 expired; onlineServers=0 2024-12-03T05:26:24,352 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'e3c6570475b6,37365,1733203533327' ***** 2024-12-03T05:26:24,352 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-03T05:26:24,352 INFO [M:0;e3c6570475b6:37365 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-03T05:26:24,352 INFO [M:0;e3c6570475b6:37365 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-03T05:26:24,352 DEBUG [M:0;e3c6570475b6:37365 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-03T05:26:24,352 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-03T05:26:24,352 DEBUG [M:0;e3c6570475b6:37365 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-03T05:26:24,352 DEBUG [master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733203536322 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.large.0-1733203536322,5,FailOnTimeoutGroup] 2024-12-03T05:26:24,352 DEBUG [master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733203536325 {}] cleaner.HFileCleaner(306): Exit Thread[master/e3c6570475b6:0:becomeActiveMaster-HFileCleaner.small.0-1733203536325,5,FailOnTimeoutGroup] 2024-12-03T05:26:24,352 INFO [M:0;e3c6570475b6:37365 {}] hbase.ChoreService(370): Chore service for: master/e3c6570475b6:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-03T05:26:24,353 INFO [M:0;e3c6570475b6:37365 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-03T05:26:24,353 DEBUG [M:0;e3c6570475b6:37365 {}] master.HMaster(1795): Stopping service threads 2024-12-03T05:26:24,353 INFO [M:0;e3c6570475b6:37365 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-03T05:26:24,353 INFO [M:0;e3c6570475b6:37365 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-03T05:26:24,354 INFO [M:0;e3c6570475b6:37365 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-03T05:26:24,354 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-03T05:26:24,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-03T05:26:24,365 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-03T05:26:24,366 DEBUG [M:0;e3c6570475b6:37365 {}] zookeeper.ZKUtil(347): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-03T05:26:24,366 WARN [M:0;e3c6570475b6:37365 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-03T05:26:24,367 INFO [M:0;e3c6570475b6:37365 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/.lastflushedseqids 2024-12-03T05:26:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741941_1121 (size=138) 2024-12-03T05:26:24,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741941_1121 (size=138) 2024-12-03T05:26:24,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741941_1121 (size=138) 2024-12-03T05:26:24,382 INFO [M:0;e3c6570475b6:37365 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-03T05:26:24,382 INFO [M:0;e3c6570475b6:37365 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-03T05:26:24,382 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-03T05:26:24,382 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:26:24,382 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:26:24,382 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-03T05:26:24,382 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:26:24,383 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=68.36 KB heapSize=83.73 KB 2024-12-03T05:26:24,397 DEBUG [M:0;e3c6570475b6:37365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fcdd369326d3472a94e0b428eb069e1f is 82, key is hbase:meta,,1/info:regioninfo/1733203537989/Put/seqid=0 2024-12-03T05:26:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741942_1122 (size=5672) 2024-12-03T05:26:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741942_1122 (size=5672) 2024-12-03T05:26:24,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741942_1122 (size=5672) 2024-12-03T05:26:24,403 INFO [M:0;e3c6570475b6:37365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fcdd369326d3472a94e0b428eb069e1f 2024-12-03T05:26:24,421 DEBUG [M:0;e3c6570475b6:37365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d229ad18522480b962a981b4019ea57 is 1077, key is \x00\x00\x00\x00\x00\x00\x00\x04/proc:d/1733203559257/Put/seqid=0 2024-12-03T05:26:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741943_1123 (size=7756) 2024-12-03T05:26:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741943_1123 (size=7756) 2024-12-03T05:26:24,426 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741943_1123 (size=7756) 2024-12-03T05:26:24,429 INFO [M:0;e3c6570475b6:37365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=67.62 KB at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d229ad18522480b962a981b4019ea57 2024-12-03T05:26:24,433 INFO [M:0;e3c6570475b6:37365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3d229ad18522480b962a981b4019ea57 2024-12-03T05:26:24,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,441 INFO [RS:0;e3c6570475b6:35421 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:26:24,441 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:35421-0x10199a715d70001, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,441 INFO [RS:0;e3c6570475b6:35421 {}] regionserver.HRegionServer(1031): Exiting; stopping=e3c6570475b6,35421,1733203534454; zookeeper connection closed. 2024-12-03T05:26:24,441 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@156d9c91 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@156d9c91 2024-12-03T05:26:24,442 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 3 regionserver(s) complete 2024-12-03T05:26:24,445 DEBUG [M:0;e3c6570475b6:37365 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a0973d47db9843b48d3b68a7799a1808 is 69, key is e3c6570475b6,35421,1733203534454/rs:state/1733203536434/Put/seqid=0 2024-12-03T05:26:24,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741944_1124 (size=5440) 2024-12-03T05:26:24,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741944_1124 (size=5440) 2024-12-03T05:26:24,451 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741944_1124 (size=5440) 2024-12-03T05:26:24,451 INFO [M:0;e3c6570475b6:37365 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=249 B at sequenceid=168 (bloomFilter=true), to=hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a0973d47db9843b48d3b68a7799a1808 2024-12-03T05:26:24,456 INFO [M:0;e3c6570475b6:37365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0973d47db9843b48d3b68a7799a1808 2024-12-03T05:26:24,457 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/fcdd369326d3472a94e0b428eb069e1f as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fcdd369326d3472a94e0b428eb069e1f 2024-12-03T05:26:24,463 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/fcdd369326d3472a94e0b428eb069e1f, entries=8, sequenceid=168, filesize=5.5 K 2024-12-03T05:26:24,465 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/3d229ad18522480b962a981b4019ea57 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d229ad18522480b962a981b4019ea57 2024-12-03T05:26:24,469 INFO [M:0;e3c6570475b6:37365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 3d229ad18522480b962a981b4019ea57 2024-12-03T05:26:24,469 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/3d229ad18522480b962a981b4019ea57, entries=17, sequenceid=168, filesize=7.6 K 2024-12-03T05:26:24,470 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/a0973d47db9843b48d3b68a7799a1808 as hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a0973d47db9843b48d3b68a7799a1808 2024-12-03T05:26:24,474 INFO [M:0;e3c6570475b6:37365 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a0973d47db9843b48d3b68a7799a1808 2024-12-03T05:26:24,474 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:43427/user/jenkins/test-data/f7f68b05-3c4a-a022-c36c-9a5db8ecb298/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/a0973d47db9843b48d3b68a7799a1808, entries=3, sequenceid=168, filesize=5.3 K 2024-12-03T05:26:24,475 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(3140): Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false 2024-12-03T05:26:24,476 INFO [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-03T05:26:24,476 DEBUG [M:0;e3c6570475b6:37365 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733203584382Disabling compacts and flushes for region at 1733203584382Disabling writes for close at 1733203584382Obtaining lock to block concurrent updates at 1733203584383 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733203584383Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=69996, getHeapSize=85680, getOffHeapSize=0, getCellsCount=195 at 1733203584383Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733203584383Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733203584383Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733203584396 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733203584396Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733203584406 (+10 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733203584420 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733203584420Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733203584433 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733203584445 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733203584445Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4b65c516: reopening flushed file at 1733203584456 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cc15b89: reopening flushed file at 1733203584464 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7265381d: reopening flushed file at 1733203584469 (+5 ms)Finished flush of dataSize ~68.36 KB/69996, heapSize ~83.44 KB/85440, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 93ms, sequenceid=168, compaction requested=false at 1733203584475 (+6 ms)Writing region close event to WAL at 1733203584476 (+1 ms)Closed at 1733203584476 2024-12-03T05:26:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45403 is added to blk_1073741830_1006 (size=83618) 2024-12-03T05:26:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:45109 is added to blk_1073741830_1006 (size=83618) 2024-12-03T05:26:24,478 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:40897 is added to blk_1073741830_1006 (size=83618) 2024-12-03T05:26:24,478 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-03T05:26:24,479 INFO [M:0;e3c6570475b6:37365 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-03T05:26:24,479 INFO [M:0;e3c6570475b6:37365 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:37365 2024-12-03T05:26:24,479 INFO [M:0;e3c6570475b6:37365 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-03T05:26:24,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,592 INFO [M:0;e3c6570475b6:37365 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-03T05:26:24,592 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:37365-0x10199a715d70000, quorum=127.0.0.1:53301, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-03T05:26:24,598 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562887 with renewLeaseKey: DEFAULT_16655 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562887 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterpartialflush-manual,16010,1733203562359/wal.1733203562887 (inode 16655) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T05:26:24,599 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testregionmadeofbulkloadedfilesonly-manual,16010,1733203552897/wal.1733203553483 with renewLeaseKey: DEFAULT_16586 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:24,599 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenviahregion-manual,16010,1733203579168/wal.1733203579508 with renewLeaseKey: DEFAULT_16767 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:24,601 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203572532 with renewLeaseKey: DEFAULT_16678 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203572532 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testreplayeditsafterabortingflush-manual,16010,1733203563108/wal.1733203572532 (inode 16678) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T05:26:24,602 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testreplayeditswrittenintowal-manual,16010,1733203542394/wal.1733203542968 with renewLeaseKey: DEFAULT_16506 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:24,604 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203573555 with renewLeaseKey: DEFAULT_16704 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203573555 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testdatalosswheninputerror-manual,16010,1733203572800/wal.1733203573555 (inode 16704) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T05:26:24,605 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testcompactedbulkloadedfiles-manual,16010,1733203573812/wal.1733203574056 with renewLeaseKey: DEFAULT_16726 java.io.IOException: stream already broken at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.endBlock(FanOutOneBlockAsyncDFSOutput.java:566) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:615) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-03T05:26:24,607 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal.1733203541707 with renewLeaseKey: DEFAULT_16485 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal.1733203541707 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit1-manual,16010,1733203540915/wal.1733203541707 (inode 16485) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T05:26:24,610 ERROR [Time-limited test {}] hdfs.DFSClient(665): Failed to close file: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal.1733203540484 with renewLeaseKey: DEFAULT_16462 java.io.FileNotFoundException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal.1733203540484 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.ipc.RemoteException.instantiateException(RemoteException.java:121) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.RemoteException.unwrapRemoteException(RemoteException.java:110) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:653) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutput.close(FanOutOneBlockAsyncDFSOutput.java:619) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hdfs.DummyDFSOutputStream.close(DummyDFSOutputStream.java:52) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeAllFilesBeingWritten(DFSClient.java:662) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.closeOutputStreams(DFSClient.java:699) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DistributedFileSystem.close(DistributedFileSystem.java:1528) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2194) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2174) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hdfs.MiniDFSCluster.shutdown(MiniDFSCluster.java:2167) ~[hadoop-hdfs-3.4.1-tests.jar:?] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniDFSCluster(HBaseTestingUtil.java:761) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1021) ~[test-classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.wal.AbstractTestWALReplay.tearDownAfterClass(AbstractTestWALReplay.java:153) ~[test-classes/:?] at org.apache.hadoop.hbase.regionserver.wal.TestAsyncWALReplay.tearDownAfterClass(TestAsyncWALReplay.java:67) ~[test-classes/:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) ~[?:?] at jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) ~[junit-4.13.2.jar:4.13.2] at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) ~[junit-4.13.2.jar:4.13.2] at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) ~[junit-4.13.2.jar:4.13.2] at java.util.concurrent.FutureTask.run(FutureTask.java:264) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.ipc.RemoteException: File does not exist: /hbase/WALs/testnameconflictwhensplit0-manual,16010,1733203538689/wal.1733203540484 (inode 16462) Holder DFSClient_NONMAPREDUCE_-1397850444_22 does not have any open files. at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.checkLease(FSNamesystem.java:3188) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFileInternal(FSDirWriteFileOp.java:703) at org.apache.hadoop.hdfs.server.namenode.FSDirWriteFileOp.completeFile(FSDirWriteFileOp.java:689) at org.apache.hadoop.hdfs.server.namenode.FSNamesystem.completeFile(FSNamesystem.java:3232) at org.apache.hadoop.hdfs.server.namenode.NameNodeRpcServer.complete(NameNodeRpcServer.java:983) at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolServerSideTranslatorPB.complete(ClientNamenodeProtocolServerSideTranslatorPB.java:649) at org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos$ClientNamenodeProtocol$2.callBlockingMethod(ClientNamenodeProtocolProtos.java) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:621) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:589) at org.apache.hadoop.ipc.ProtobufRpcEngine2$Server$ProtoBufRpcInvoker.call(ProtobufRpcEngine2.java:573) at org.apache.hadoop.ipc.RPC$Server.call(RPC.java:1227) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1246) at org.apache.hadoop.ipc.Server$RpcCall.run(Server.java:1169) at java.base/java.security.AccessController.doPrivileged(AccessController.java:712) at java.base/javax.security.auth.Subject.doAs(Subject.java:439) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) at org.apache.hadoop.ipc.Server$Handler.run(Server.java:3198) at org.apache.hadoop.ipc.Client.getRpcResponse(Client.java:1584) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1529) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.Client.call(Client.java:1426) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:258) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.ipc.ProtobufRpcEngine2$Invoker.invoke(ProtobufRpcEngine2.java:139) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy43.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.lambda$complete$13(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.ipc.internal.ShadedProtobufHelper.ipc(ShadedProtobufHelper.java:160) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hdfs.protocolPB.ClientNamenodeProtocolTranslatorPB.complete(ClientNamenodeProtocolTranslatorPB.java:532) ~[hadoop-hdfs-client-3.4.1.jar:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invokeMethod(RetryInvocationHandler.java:437) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeMethod(RetryInvocationHandler.java:170) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invoke(RetryInvocationHandler.java:162) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler$Call.invokeOnce(RetryInvocationHandler.java:100) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.io.retry.RetryInvocationHandler.invoke(RetryInvocationHandler.java:366) ~[hadoop-common-3.4.1.jar:?] at jdk.proxy2.$Proxy44.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at jdk.internal.reflect.GeneratedMethodAccessor3.invoke(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) ~[?:?] at java.lang.reflect.Method.invoke(Method.java:568) ~[?:?] at org.apache.hadoop.hbase.fs.HFileSystem$1.invoke(HFileSystem.java:363) ~[classes/:?] at jdk.proxy2.$Proxy47.complete(Unknown Source) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputHelper.completeFile(FanOutOneBlockAsyncDFSOutputHelper.java:646) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 25 more 2024-12-03T05:26:24,617 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@420bc738{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:26:24,619 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5546381a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:26:24,619 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:26:24,620 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@228ffa29{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:26:24,620 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1e1a8b20{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,STOPPED} 2024-12-03T05:26:24,622 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T05:26:24,622 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T05:26:24,622 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T05:26:24,622 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1175346662-172.17.0.3-1733203526961 (Datanode Uuid 627416e0-b63c-4031-8f58-2014fcb13715) service to localhost/127.0.0.1:43427 2024-12-03T05:26:24,623 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data5/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,624 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data6/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,624 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T05:26:24,626 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@64052af6{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:26:24,626 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@645d24a7{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:26:24,626 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:26:24,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7c7d32f8{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:26:24,627 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7acb2c1e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,STOPPED} 2024-12-03T05:26:24,628 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T05:26:24,628 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T05:26:24,628 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1175346662-172.17.0.3-1733203526961 (Datanode Uuid 559ab081-2cbf-42fa-87fa-76b5b8954585) service to localhost/127.0.0.1:43427 2024-12-03T05:26:24,628 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T05:26:24,628 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data3/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,628 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data4/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,629 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T05:26:24,631 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@501095cc{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-03T05:26:24,631 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2b37aa87{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:26:24,631 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:26:24,631 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@751d2fa4{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:26:24,631 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3161f7a1{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,STOPPED} 2024-12-03T05:26:24,632 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-03T05:26:24,632 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-03T05:26:24,632 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-03T05:26:24,632 WARN [BP-1175346662-172.17.0.3-1733203526961 heartbeating to localhost/127.0.0.1:43427 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1175346662-172.17.0.3-1733203526961 (Datanode Uuid 95d316af-ec37-443a-97a2-cfb6e6f154f4) service to localhost/127.0.0.1:43427 2024-12-03T05:26:24,633 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data1/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,633 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/cluster_aad0226a-3ce1-886e-b139-79ac21067d87/data/data2/current/BP-1175346662-172.17.0.3-1733203526961 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-03T05:26:24,633 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-03T05:26:24,640 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4c03e044{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-03T05:26:24,641 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@698721f1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-03T05:26:24,641 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-03T05:26:24,641 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@652099c6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-03T05:26:24,641 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@264e890{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/546037fb-b90a-1f17-0be9-230bb9898085/hadoop.log.dir/,STOPPED} 2024-12-03T05:26:24,652 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-03T05:26:24,698 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down